Public/removeads
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve linting by remove trailing spaces.

Browse Source
This commit is contained in:
Frédéric Tronel
2025-10-25 16:45:20 +02:00
parent c3943ff70e
commit ddec8633e3
1 changed files with 79 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
removeads.py
View File

@@ -93,11 +93,11 @@ def getTesseractSupportedLang(tesseract):
def getFrameRate(ffprobe, inputFile):
logger = logging.getLogger(__name__)
infd = inputFile.fileno()
lseek(infd, 0, SEEK_SET)
set_inheritable(infd, True)
meanDuration = 0.
nbFrames1 = 0
nbFrames2 = 0
@@ -105,7 +105,7 @@ def getFrameRate(ffprobe, inputFile):
minTs = None
maxTs = None
interlaced = False
params = [ffprobe, '-loglevel', 'quiet', '-select_streams', 'v', '-show_frames',
'-read_intervals', '00%+30', '-of', 'json', '/proc/self/fd/%d' % infd]
env = {**os.environ, 'LANG': 'C'}
@@ -133,13 +133,13 @@ def getFrameRate(ffprobe, inputFile):
nbFrames2+=1
else:
return None
ffprobe.wait()
if ffprobe.returncode != 0:
logger.error("ffprobe returns an error code: %d", ffprobe.returncode)
return None
frameRate1 = nbFrames1/(maxTs-minTs)
frameRate2 = nbFrames2 / meanDuration
@@ -177,34 +177,34 @@ def getSubTitlesTracks(ffprobe, mkvPath):
else:
l = tracks[lang]
l.append(index)
tracks[lang] = l
tracks[lang] = l
else:
return None
ffprobe.wait()
if ffprobe.returncode != 0:
logger.error("ffprobe returns an error code: %d", ffprobe.returncode)
return None
return tracks
def extractSRT(mkvextract, fileName, subtitles, langs):
logger = logging.getLogger(__name__)
params = [mkvextract, fileName, 'tracks']
res = []
for lang in subtitles:
iso = Lang(lang)
if iso in langs:
ocrlang = langs[iso]
else:
logger.warning("Language not supported by Tesseract: %s", iso.name)
ocrlang ='osd'
if len(subtitles[lang]) == 1:
params.append('%d:%s' % (subtitles[lang][0], lang))
res.append(('%s.idx' % lang, '%s.sub' % lang, lang, ocrlang))
@@ -230,9 +230,9 @@ def extractSRT(mkvextract, fileName, subtitles, langs):
pb.update(100-pb.n)
pb.refresh()
pb.close()
extract.wait()
# mkvextract returns 0, 1 or 2 as error code.
if extract.returncode == 0:
logger.info('Subtitle tracks were succesfully extracted.')
@@ -243,11 +243,11 @@ def extractSRT(mkvextract, fileName, subtitles, langs):
else:
logger.error('Mkvextract returns an error code: %d', extract.returncode)
return None
def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False):
logger = logging.getLogger(__name__)
res = []
for idxName, subName, lang, iso in idxs:
srtname = '%s.srt' % os.path.splitext(idxName)[0]
# Tesseract seems to recognize the three dots ... as "su"
@@ -263,7 +263,7 @@ def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False):
write(srtfd, '...'.encode(encoding='UTF-8'))
else:
write(srtfd, line.encode(encoding='UTF-8'))
m = re.match(timestamps, line)
if m!=None:
hours = int(m.group('hours'))
@@ -274,7 +274,7 @@ def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False):
pb.update()
status = ocr.wait()
if status != 0:
logger.error('OCR failed with status code: %d', status)
@@ -286,7 +286,7 @@ def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False):
return None
lseek(srtfd, 0, SEEK_SET)
srtLength = fstat(srtfd).st_size
srtLength = fstat(srtfd).st_size
buf = read(srtfd, srtLength)
outfd = dumpSrt.fileno()
pos = 0
@@ -319,14 +319,14 @@ class SupportedFormat(IntEnum):
else:
return 'Unsupported format'
# Extract SPS/PPS
# Extract SPS/PPS
# https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390
# ffmpeg -i <InputFile (before concatenation)> -c:v copy -an -sn -bsf:v trace_headers -t 0.01 -report -loglevel 0 -f null -
# Found codec private data using mkvinfo
def getCodecPrivateDataFromMKV(mkvinfo, inputFile):
logger = logging.getLogger(__name__)
infd = inputFile.fileno()
lseek(infd, 0, SEEK_SET)
set_inheritable(infd, True)
@@ -334,7 +334,7 @@ def getCodecPrivateDataFromMKV(mkvinfo, inputFile):
env = {**os.environ, 'LANG': 'C'}
# Output example
# Codec's private data: size 48 (H.264 profile: High @L4.0) hexdump 01 64 00 28 ff e1 00 1b 67 64 00 28 ac d9 40 78 04 4f dc d4 04 04 05 00 00 92 ef 00 1d ad a6 1f 16 2d 96 01 00 06 68 fb a3 cb 22 c0 fd f8 f8 00 at 406 size 51 data size 48
with Popen([mkvinfo, '-z', '-X', '-P', '/proc/self/fd/%d' % infd ], stdout=PIPE, close_fds=False, env=env) as mkvinfo:
out, _ = mkvinfo.communicate()
out = out.decode('utf8')
@@ -349,7 +349,7 @@ def getCodecPrivateDataFromMKV(mkvinfo, inputFile):
found = True
mkvinfo.wait()
break
if found:
lseek(infd, position, SEEK_SET)
data = read(infd, size)
@@ -601,7 +601,7 @@ def writeScalingList(buf, bitPosition, size, matrix, optimized=False):
# deltas.append(0)
# for delta in deltas:
# bitPosition = writeSignedExpGolomb(buf, bitPosition, delta)
return bitPosition
@dataclass
@@ -1209,7 +1209,7 @@ class PPS:
bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v)
elif self.slice_group_map_type == 2:
for i in range(0, self.num_slice_groups_minus1):
v = self.top_left[i]
v = self.top_left[i]
bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v)
v = self.bottom_right[i]
bitPosition = writeUnsignedExpGolomb(buf, bitPosition, v)
@@ -1255,7 +1255,7 @@ class PPS:
else:
logger.info("Writing matrix: %s" % matrix)
bitPosition = writeScalingList(buf, bitPosition, 64, matrix)
bitPosition = writeSignedExpGolomb(buf, bitPosition, self.second_chroma_qp_index_offset)
bitPosition = writeSignedExpGolomb(buf, bitPosition, self.second_chroma_qp_index_offset)
bitPosition = writeRBSPTrailingBits(buf, bitPosition)
@@ -1267,7 +1267,7 @@ class AVCDecoderConfiguration:
AVCProfileIndication:int=0 # u(8)
profile_compatibility:int=0 # u(8)
AVCLevelIndication:int=0 # u(8)
lengthSizeMinusOne:int=0 # u(2) (0,1 or 3)
lengthSizeMinusOne:int=0 # u(2) (0,1 or 3)
numOfSequenceParameterSets:int=0 # u(5)
sps:dict = field(default_factory=dict)
numOfPictureParameterSets:int=0 #u(8)
@@ -1413,7 +1413,7 @@ class AVCDecoderConfiguration:
bitPosition = writeBits(buf, bitPosition, self.bit_depth_chroma_minus8, 3)
bitPosition = writeByte(buf, bitPosition, self.numOfSequenceParameterSetExt)
for i in range(0, self.numOfSequenceParameterSetExt):
# TODO: dump SPSextended
# TODO: dump SPSextended
logger.error('Dumping SPS extended not yet implemented')
pass
@@ -1477,7 +1477,7 @@ def parseCodecPrivate(codecPrivateData):
for i in range(0, nbZeroes):
length*=256
length+=(codecPrivateData[3+i])
bytePosition = 3+nbZeroes
bytePosition = 3+nbZeroes
avcconfig = AVCDecoderConfiguration()
avcconfig.fromBytes(codecPrivateData[bytePosition:])
@@ -1586,10 +1586,10 @@ def parseMKVTree(mkvinfo, inputFile):
# cf http://matroska-org.github.io/libebml/specs.html
# It is a Type, Length, Value (TLV) kind of binary file.
# Types are encoded as follows:
# 1xxx xxxx - Class A IDs (2^7 -1 possible values)
# 1xxx xxxx - Class A IDs (2^7 -1 possible values)
# 01xx xxxx xxxx xxxx - Class B IDs (2^14-1 possible values)
# 001x xxxx xxxx xxxx xxxx xxxx - Class C IDs (2^21-1 possible values)
# 0001 xxxx xxxx xxxx xxxx xxxx xxxx xxxx - Class D IDs (2^28-1 possible values)
# 001x xxxx xxxx xxxx xxxx xxxx - Class C IDs (2^21-1 possible values)
# 0001 xxxx xxxx xxxx xxxx xxxx xxxx xxxx - Class D IDs (2^28-1 possible values)
# Lengths are encoded as follows:
# 1xxx xxxx - value 0 to 2^7-2
# 01xx xxxx xxxx xxxx - value 0 to 2^14-2
@@ -1609,7 +1609,7 @@ def getEBMLLength(length):
elif length <= 2**14-2:
size = 2
elif length <= 2**21-2:
size = 3
size = 3
elif length <= 2**28-2:
size = 4
elif length <= 2**35-2:
@@ -1639,7 +1639,7 @@ def dumpCodecPrivateData(AVCDecoderConfiguration):
# Code private element
res.extend(b'\x63\xA2')
buf = AVCDecoderConfiguration.toBytes()
logger.debug('AVC configuration bitstream: %s (length: %d))' % (hexdump.dump(buf, sep=':'), len(buf)))
logger.debug('AVC configuration bitstream: %s (length: %d))' % (hexdump.dump(buf, sep=':'), len(buf)))
EMBLlength = getEBMLLength(len(buf))
logger.debug('EMBL encoded length: %s' % (hexdump.dump(EMBLlength, sep=':')))
@@ -1672,13 +1672,13 @@ def changeEBMLElementSize(inputFile, position, addendum):
logger.error('Size of element type cannot be determined: %d', elementType)
exit(-1)
# We seek to size
# We seek to size
position+=typeSize
lseek(infd, position, SEEK_SET)
buf = read(infd, 1)
sizeHead = int.from_bytes(buf, byteorder='big')
logger.info('First byte of size: %x' % sizeHead)
logger.info('First byte of size: %x' % sizeHead)
mask=128
found = False
for i in range(1,9):
@@ -1714,7 +1714,7 @@ def changeEBMLElementSize(inputFile, position, addendum):
exit(-1)
# The difference of length between old size field and new one.
delta = sizeOfNewEncodedSize - sizeOfDataSize
fileLength = fstat(infd).st_size
fileLength = fstat(infd).st_size
# We seek after actual length field
lseek(infd, position+sizeOfDataSize, SEEK_SET)
# We read the rest of file
@@ -1796,7 +1796,7 @@ def changeCodecPrivateData(mkvinfo, inputFile, codecData):
logger.info(keys)
delta = futureLength-currentLength
# if there is no modification of the private codec data, no need to change anything.
# if there is no modification of the private codec data, no need to change anything.
if delta != 0:
for i in range(0, len(keys)-1):
keys.pop()
@@ -2034,9 +2034,9 @@ def ffmpegConvert(ffmpeg, ffprobe, inputFile, inputFormat, outputFile, outputFor
else:
log = [ '-loglevel', 'quiet' ]
params = [ffmpeg, '-y',]+log+['-progress', '/dev/stdout', '-canvas_size', '%dx%d' % (width, height), '-f', inputFormat, '-i', '/proc/self/fd/%d' % infd,
params = [ffmpeg, '-y',]+log+['-progress', '/dev/stdout', '-canvas_size', '%dx%d' % (width, height), '-f', inputFormat, '-i', '/proc/self/fd/%d' % infd,
'-map', '0:v', '-map', '0:a']
if subtitles:
if subtitles:
params.extend(['-map', '0:s'])
params.extend(['-bsf:v', 'h264_mp4toannexb,dump_extra=freq=keyframe', '-vcodec', 'copy', '-acodec', 'copy'])
if subtitles:
@@ -2089,11 +2089,11 @@ def getFramesInStream(ffprobe, inputFile, begin, end, streamKind, subStreamId=0)
logger = logging.getLogger(__name__)
infd = inputFile.fileno()
set_inheritable(infd, True)
command = [ffprobe, '-loglevel', 'quiet', '-read_intervals', ('%s%%%s' %(begin, end)), '-show_entries', 'frame',
command = [ffprobe, '-loglevel', 'quiet', '-read_intervals', ('%s%%%s' %(begin, end)), '-show_entries', 'frame',
'-select_streams', '%s:%d' % (streamKind, subStreamId), '-of', 'json', '/proc/self/fd/%d' % infd]
logger.debug('Executing: %s' % command)
logger.debug('Executing: %s', command)
with Popen(command, stdout=PIPE, close_fds=False) as ffprobe:
out, _ = ffprobe.communicate()
frames = json.load(BytesIO(out))
@@ -2101,7 +2101,7 @@ def getFramesInStream(ffprobe, inputFile, begin, end, streamKind, subStreamId=0)
if status != 0:
logger.error('ffprobe failed with status code: %d' % status)
return None
# Sort frames by timestamp
tmp = {}
if 'frames' in frames:
@@ -2112,33 +2112,33 @@ def getFramesInStream(ffprobe, inputFile, begin, end, streamKind, subStreamId=0)
return None
if begin <= ts and ts <= end:
tmp[ts]=frame
res = []
for ts in sorted(tmp):
res.append(tmp[ts])
return res
else:
logger.error('Impossible to retrieve frames inside file around [%s,%s]' % (begin, end))
return None
# TODO:
# TODO:
def getNearestIDRFrame(ffprobe, inputFile, timestamp, before=True, delta=timedelta(seconds=2)):
logger = logging.getLogger(__name__)
zero = timedelta()
tbegin = timestamp-delta
tend = timestamp+delta
if tbegin < zero:
tbegin = zero
infd = inputFile.fileno()
set_inheritable(infd, True)
logger.debug('Looking for IDR frame in [%s, %s]', tbegin, tend)
idrs = []
# Retains only IDR frame
with Popen([ffprobe, '-loglevel', 'quiet', '-read_intervals', ('%s%%%s' %(tbegin, tend)), '-skip_frame', 'nokey', '-show_entries', 'frame', '-select_streams', 'v:0', '-of', 'json', '/proc/self/fd/%d' % infd], stdout=PIPE, close_fds=False) as ffprobe:
out, _ = ffprobe.communicate()
@@ -2159,16 +2159,13 @@ def getNearestIDRFrame(ffprobe, inputFile, timestamp, before=True, delta=timedel
else:
logger.error('Impossible to retrieve IDR frames inside file around [%s,%s]', tbegin, tend)
return None
def getNearestIFrame(ffprobe, inputFile, timestamp, before=True, deltaMax=timedelta(seconds=15)):
logger = logging.getLogger(__name__)
infd = inputFile.fileno()
set_inheritable(infd, True)
delta = timedelta(seconds=1)
iframe = None
@@ -2184,26 +2181,26 @@ def getNearestIFrame(ffprobe, inputFile, timestamp, before=True, deltaMax=timede
tend = timestamp
if tbegin < zero:
tbegin = zero
logger.debug('Looking for an iframe in [%s, %s]' % (tbegin, tend))
logger.debug('Looking for an iframe in [%s, %s]', tbegin, tend)
frames = getFramesInStream(ffprobe, inputFile=inputFile, begin=tbegin, end=tend, streamKind='v')
if frames is None:
logger.debug('Found no frame in [%s, %s]' % (tbegin, tend))
logger.debug('Found no frame in [%s, %s]', tbegin, tend)
delta+=timedelta(seconds=1)
continue
iframes = []
for frame in frames:
if frame['pict_type'] == 'I':
iframes.append(frame)
found = False
for frame in iframes:
ts = getTSFrame(frame)
if ts is None:
logger.warning('I-frame with no timestamp: %s' % frame)
continue
if before and ts <= timestamp:
found = True
iframe = frame
@@ -2218,7 +2215,7 @@ def getNearestIFrame(ffprobe, inputFile, timestamp, before=True, deltaMax=timede
else:
delta+=timedelta(seconds=1)
continue
if iframe is not None:
its = getTSFrame(iframe)
nbFrames = 0
@@ -2296,7 +2293,7 @@ def extractPictures(ffmpeg, inputFile, begin, nbFrames, width=640, height=480):
length = imageLength*nbFrames
logger.debug("Estimated length: %d" % length)
command = [ffmpeg, '-loglevel', 'quiet' ,'-y', '-ss', '%s'%begin, '-i', '/proc/self/fd/%d' % infd, '-s', '%dx%d'%(width, height),
command = [ffmpeg, '-loglevel', 'quiet' ,'-y', '-ss', '%s'%begin, '-i', '/proc/self/fd/%d' % infd, '-s', '%dx%d'%(width, height),
'-vframes', '%d'%nbFrames, '-c:v', 'ppm', '-f', 'image2pipe', '/proc/self/fd/%d' % outfd ]
logger.debug('Executing: %s', command)
@@ -2310,7 +2307,7 @@ def extractPictures(ffmpeg, inputFile, begin, nbFrames, width=640, height=480):
lseek(outfd, 0, SEEK_SET)
images = read(outfd,length)
if len(images) != length:
logger.error("Received %d bytes but %d were expected." % (len(images), length))
logger.error("Received %d bytes but %d were expected." % (len(images), length))
return None, None
lseek(outfd, 0, SEEK_SET)
@@ -2327,7 +2324,7 @@ def extractSound(ffmpeg, inputFile, begin, outputFileName, packetDuration, subCh
sound = bytes()
length = int(nbChannels*sampleRate*4*nbPackets*packetDuration/1000)
command = [ffmpeg, '-y', '-loglevel', 'quiet', '-ss', '%s'%begin, '-i', '/proc/self/fd/%d' % infd, '-frames:a:%d' % subChannel, '%d' % (nbPackets+1),
command = [ffmpeg, '-y', '-loglevel', 'quiet', '-ss', '%s'%begin, '-i', '/proc/self/fd/%d' % infd, '-frames:a:%d' % subChannel, '%d' % (nbPackets+1),
'-c:a', 'pcm_s32le', '-sample_rate', '%d' % sampleRate, '-channels', '%d' % nbChannels, '-f', 's32le', '/proc/self/fd/%d' % outfd]
logger.debug('Executing: %s', command)
@@ -2349,7 +2346,7 @@ def extractSound(ffmpeg, inputFile, begin, outputFileName, packetDuration, subCh
def dumpPPM(pictures, prefix, temporaries):
logger = logging.getLogger(__name__)
# "P6\nWIDTH HEIGHT\n255\n"
# "P6\nWIDTH HEIGHT\n255\n"
pos = 0
picture = 0
@@ -2362,7 +2359,7 @@ def dumpPPM(pictures, prefix, temporaries):
dimensions = header.readline().decode('utf8')
maxvalue = header.readline().decode('utf8')
if magic == 'P6\n':
pattern = re.compile('^(?P<width>[0-9]+) (?P<height>[0-9]+)\n$')
pattern = re.compile('^(?P<width>[0-9]+) (?P<height>[0-9]+)\n$')
m = pattern.match(dimensions)
if m is not None:
width = int(m['width'])
@@ -2372,7 +2369,7 @@ def dumpPPM(pictures, prefix, temporaries):
return
else:
logger.error('Not a PPM picture')
return
return
headerLen=2+1+ceil(log(width, 10))+1+ceil(log(height, 10))+1+3+1
try:
@@ -2394,7 +2391,7 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref
logger = logging.getLogger(__name__)
# The command line for encoding only video track
videoEncoderParams = [ ffmpeg, '-y', '-loglevel', 'quiet']
videoEncoderParams = [ ffmpeg, '-y', '-loglevel', 'quiet']
videoInputParams = []
videoCodecParams = []
@@ -2419,7 +2416,7 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref
colorTransfer = stream['color_transfer']
colorPrimaries = stream['color_primaries']
level = int(stream['level'])
level = '%d.%d' % (floor(level/10), level%10)
level = '%d.%d' % (floor(level/10), level%10)
chromaLocation = stream['chroma_location']
fieldOrder = stream
interlacedOptions = []
@@ -2627,7 +2624,7 @@ def mergeMKVs(mkvmerge, inputs, outputName, concatenate=True, timestamps=None):
fd = mkv.fileno()
fds.append(fd)
set_inheritable(fd, True)
# If we pass a timestamps file associated with the considered track, use it.
# If we pass a timestamps file associated with the considered track, use it.
if timestamps is not None and partNum in timestamps:
tsfd = timestamps[partNum].fileno()
lseek(tsfd, 0, SEEK_SET)
@@ -2825,7 +2822,7 @@ def concatenateH264Parts(h264parts, output):
totalLength = 0
for h264 in h264parts:
fd = h264.fileno()
totalLength += fstat(fd).st_size
totalLength += fstat(fd).st_size
logger.info('Total length: %d', totalLength)
@@ -2843,7 +2840,7 @@ def concatenateH264Parts(h264parts, output):
pos = 0
while pos < len(buf):
nbBytes = write(outfd, buf[pos:])
pb.update(nbBytes)
pb.update(nbBytes)
pos += nbBytes
def concatenateH264TSParts(h264TSParts, output):
@@ -3001,7 +2998,7 @@ def main():
if formatOfFile == SupportedFormat.TS:
logger.info("Converting TS to MP4 (to fix timestamps).")
try:
with open(mp4filename, 'w+') as mp4:
with open(mp4filename, 'w+') as mp4:
ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], inputFile, 'mpegts', mp4, 'mp4', duration)
temporaries.append(mp4)
logger.info("Converting MP4 to MKV.")
@@ -3094,7 +3091,7 @@ def main():
# Si la trame de début est déjà 'I', il n'y a rien à faire.
# Sinon on extrait les trames 'B' ou 'P' depuis le début jusqu'à la trame 'I' non incluse.
# Si la trame de fin précède une trame I, on n'a rien à faire.
# Sinon on extrait toutes les trames depuis la dernière trame I jusqu'à la trame de fin.
# Sinon on extrait toutes les trames depuis la dernière trame I jusqu'à la trame de fin.
partnum = partnum + 1
@@ -3226,7 +3223,7 @@ def main():
checks.append(pos)
# When using coarse option there is a single AVC configuration.
for avcConfig in otherAvcConfigs:
for avcConfig in otherAvcConfigs:
mainAvcConfig.merge(avcConfig)
logger.debug('Merged AVC configuration: %s', mainAvcConfig)
@@ -3279,7 +3276,7 @@ def main():
finalWithVideo = mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=[fullH264, finalNoVideo], outputName=finalWithVideoName, concatenate=False, timestamps={0: fullH264TS})
finalCodecPrivateData = dumpCodecPrivateData(mainAvcConfig)
logger.debug('Final codec private data: %s' % hexdump.dump(finalCodecPrivateData, sep=':'))
logger.info('Changing codec private data with the new one.')
logger.info('Changing codec private data with the new one.')
changeCodecPrivateData(paths['mkvinfo'], finalWithVideo, finalCodecPrivateData)
if args.srt: