Public/removeads
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Still more linting (encoding for open, variables with snake case).

Browse Source
This commit is contained in:
Frédéric Tronel
2025-10-28 10:55:47 +01:00
parent 960de11b1b
commit 75f227786f
1 changed files with 79 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
removeads.py
View File

@@ -294,20 +294,19 @@ def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False):
if dumpMemFD: if dumpMemFD:
try: try:
dump_srt = open(srtname,'w') with open(srtname,'w', encoding='utf8') as dump_srt:
lseek(srtfd, 0, SEEK_SET)
srt_length = fstat(srtfd).st_size
buf = read(srtfd, srt_length)
outfd = dump_srt.fileno()
pos = 0
while pos < srt_length:
pos+=write(outfd, buf[pos:])
temporaries.append(dump_srt)
except IOError: except IOError:
logger.error('Impossible to create file: %s', srtname) logger.error('Impossible to create file: %s', srtname)
return None return None
lseek(srtfd, 0, SEEK_SET)
srt_length = fstat(srtfd).st_size
buf = read(srtfd, srt_length)
outfd = dump_srt.fileno()
pos = 0
while pos < srt_length:
pos+=write(outfd, buf[pos:])
temporaries.append(dump_srt)
srt_length = fstat(srtfd).st_size srt_length = fstat(srtfd).st_size
if srt_length > 0: if srt_length > 0:
@@ -2471,7 +2470,7 @@ def dumpPPM(pictures, prefix, temporaries):
header = BytesIO(pictures[pos:]) header = BytesIO(pictures[pos:])
magic = header.readline().decode('utf8') magic = header.readline().decode('utf8')
dimensions = header.readline().decode('utf8') dimensions = header.readline().decode('utf8')
max_value = header.readline().decode('utf8') max_value = int(header.readline().decode('utf8'))
if magic == 'P6\n': if magic == 'P6\n':
pattern = re.compile('^(?P<width>[0-9]+) (?P<height>[0-9]+)\n$') pattern = re.compile('^(?P<width>[0-9]+) (?P<height>[0-9]+)\n$')
m = pattern.match(dimensions) m = pattern.match(dimensions)
@@ -2491,7 +2490,7 @@ def dumpPPM(pictures, prefix, temporaries):
header_len=2+1+ceil(log(width, 10))+1+ceil(log(height, 10))+1+3+1 header_len=2+1+ceil(log(width, 10))+1+ceil(log(height, 10))+1+3+1
try: try:
out = open(filename, 'w') out = open(filename, 'w', encoding='utf8')
outfd = out.fileno() outfd = out.fileno()
except IOError: except IOError:
logger.error('Impossible to create file: %s', filename) logger.error('Impossible to create file: %s', filename)
@@ -2630,7 +2629,7 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref
if dumpMemFD: if dumpMemFD:
try: try:
output = open(tmpname,'w') output = open(tmpname,'w', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file: %s', tmpname) logger.error('Impossible to create file: %s', tmpname)
return None return None
@@ -2722,7 +2721,7 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref
h264_ts_filename = f'{filesPrefix}-ts.txt' h264_ts_filename = f'{filesPrefix}-ts.txt'
try: try:
h264_ts_output = open(h264_ts_filename,'w+') h264_ts_output = open(h264_ts_filename,'w+', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file: %s', h264_ts_filename) logger.error('Impossible to create file: %s', h264_ts_filename)
return None return None
@@ -2752,7 +2751,7 @@ def mergeMKVs(mkvmerge, inputs, outputName, concatenate=True, timestamps=None):
fds = [] fds = []
try: try:
out = open(outputName, 'w+') out = open(outputName, 'w+', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file: %s', outputName) logger.error('Impossible to create file: %s', outputName)
return None return None
@@ -2770,7 +2769,7 @@ def mergeMKVs(mkvmerge, inputs, outputName, concatenate=True, timestamps=None):
first = True first = True
partnum = 0 partnum = 0
for mkv in inputs: for mkv in inputs:
if mkv !=None: if mkv is not None:
fd = mkv.fileno() fd = mkv.fileno()
fds.append(fd) fds.append(fd)
set_inheritable(fd, True) set_inheritable(fd, True)
@@ -2929,7 +2928,7 @@ def remuxSRTSubtitles(mkvmerge, inputFile, outputFileName, subtitles):
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
try: try:
out = open(outputFileName, 'w') out = open(outputFileName, 'w', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file: %s', outputFileName) logger.error('Impossible to create file: %s', outputFileName)
return None return None
@@ -3039,7 +3038,7 @@ def doCoarseProcessing(ffmpeg, ffprobe, mkvmerge, inputFile, begin, end, nbFrame
internal_mkv_name = f'{filesPrefix}.mkv' internal_mkv_name = f'{filesPrefix}.mkv'
try: try:
internal_mkv = open(internal_mkv_name, 'w+') internal_mkv = open(internal_mkv_name, 'w+', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file: %s', internal_mkv_name) logger.error('Impossible to create file: %s', internal_mkv_name)
exit(-1) exit(-1)
@@ -3133,7 +3132,7 @@ def main():
mkvfilename = basename+'.mkv' mkvfilename = basename+'.mkv'
try: try:
input_file = open(args.input_file, mode='r') input_file = open(args.input_file, mode='r', encoding='utf8')
except IOError: except IOError:
logger.error("Impossible to open %s", args.input_file) logger.error("Impossible to open %s", args.input_file)
exit(-1) exit(-1)
@@ -3170,13 +3169,13 @@ def main():
if format_of_file == SupportedFormat.TS: if format_of_file == SupportedFormat.TS:
logger.info("Converting TS to MP4 (to fix timestamps).") logger.info("Converting TS to MP4 (to fix timestamps).")
try: try:
with open(mp4filename, 'w+') as mp4: with open(mp4filename, 'w+', encoding='utf8') as mp4:
ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], input_file, 'mpegts', mp4, 'mp4', ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], input_file, 'mpegts', mp4, 'mp4',
duration) duration)
temporaries.append(mp4) temporaries.append(mp4)
logger.info("Converting MP4 to MKV.") logger.info("Converting MP4 to MKV.")
try: try:
mkv = open(mkvfilename, 'w+') mkv = open(mkvfilename, 'w+', encoding='utf8')
except IOError: except IOError:
logger.error('') logger.error('')
@@ -3190,7 +3189,7 @@ def main():
elif format_of_file == SupportedFormat.MP4: elif format_of_file == SupportedFormat.MP4:
logger.info("Converting MP4 to MKV") logger.info("Converting MP4 to MKV")
try: try:
mkv = open(mkvfilename, 'w+') mkv = open(mkvfilename, 'w+', encoding='utf8')
except IOError: except IOError:
logger.error('') logger.error('')
ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], input_file, 'mp4', mkv, 'matroska', ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], input_file, 'mp4', mkv, 'matroska',
@@ -3235,31 +3234,31 @@ def main():
logger.debug('AVC configuration: %s', main_avc_config) logger.debug('AVC configuration: %s', main_avc_config)
# We check if the parse and dump operations are idempotent. # We check if the parse and dump operations are idempotent.
privateData = dumpCodecPrivateData(main_avc_config) private_data = dumpCodecPrivateData(main_avc_config)
logger.debug('Redump AVC configuration: %s', hexdump.dump(privateData, sep=':')) logger.debug('Redump AVC configuration: %s', hexdump.dump(private_data, sep=':'))
# In rare occasion, the PPS has trailing zeroes that do not seem to be related to useful data # In rare occasion, the PPS has trailing zeroes that do not seem to be related to useful data
# but they differ from the private data we generate that do not contain them. # but they differ from the private data we generate that do not contain them.
# In that case we try to redecode our own private data to see if both AVC configurations are # In that case we try to redecode our own private data to see if both AVC configurations are
# the same. # the same.
if main_codec_private_data != privateData: if main_codec_private_data != private_data:
logger.warning('Difference detected in bitstream !!') logger.warning('Difference detected in bitstream !!')
isoAvcConfig = parseCodecPrivate(privateData) iso_avc_config = parseCodecPrivate(private_data)
logger.debug('Reread AVC configuration: %s', isoAvcConfig) logger.debug('Reread AVC configuration: %s', iso_avc_config)
# If there exists a difference between our own reconstructed AVC configuration and the # If there exists a difference between our own reconstructed AVC configuration and the
# original one, we abandon # original one, we abandon
if isoAvcConfig != main_avc_config: if iso_avc_config != main_avc_config:
logger.error('AVC configurations are different: %s\n%s\n', main_avc_config, isoAvcConfig) logger.error('AVC configurations are different: %s\n%s\n', main_avc_config, iso_avc_config)
exit(-1) exit(-1)
# Pour chaque portion # Pour chaque portion
partnum = 0 partnum = 0
mkvparts = [] mkvparts = []
h264parts = [] h264parts = []
h264TS = [] h264_ts = []
checks = [] checks = []
pos = timedelta() pos = timedelta()
otherAvcConfigs = [] other_avc_configs = []
for ts1, ts2 in parts: for ts1, ts2 in parts:
# TODO: translate comment in english # TODO: translate comment in english
@@ -3278,32 +3277,34 @@ def main():
partnum = partnum + 1 partnum = partnum + 1
# Get the nearest I-frame whose timestamp is greater or equal to the beginning. # Get the nearest I-frame whose timestamp is greater or equal to the beginning.
headFrames = getNearestIFrame(paths['ffprobe'], mkv, ts1, before=False) head_frames = getNearestIFrame(paths['ffprobe'], mkv, ts1, before=False)
if headFrames is None: if head_frames is None:
logger.error('Impossible to retrieve I-frame')
exit(-1) exit(-1)
# Get the nearest I-frame whose timestamp ... # Get the nearest I-frame whose timestamp ...
# TODO: wrong here ... # TODO: wrong here ...
tailFrames = getNearestIFrame(paths['ffprobe'], mkv, ts2, before=True) tail_frames = getNearestIFrame(paths['ffprobe'], mkv, ts2, before=True)
if tailFrames is None: if tail_frames is None:
logger.error('Impossible to retrieve I-frame')
exit(-1) exit(-1)
nbHeadFrames, headIFrame = headFrames nb_head_frames, head_iframe = head_frames
nbTailFrames, tailIFrame = tailFrames nb_tail_frames, tail_iframe = tail_frames
logger.info("Found %d frames between beginning of current part and first I-frame", logger.info("Found %d frames between beginning of current part and first I-frame",
nbHeadFrames) nb_head_frames)
logger.info("Found %d frames between last I-frame and end of current part", logger.info("Found %d frames between last I-frame and end of current part",
nbTailFrames) nb_tail_frames)
headIFrameTS = getTSFrame(headIFrame) head_iframe_ts = getTSFrame(head_iframe)
if headIFrameTS is None: if head_iframe_ts is None:
exit(-1) exit(-1)
tailIFrameTS = getTSFrame(tailIFrame) tail_iframe_ts = getTSFrame(tail_iframe)
if tailIFrameTS is None: if tail_iframe_ts is None:
exit(-1) exit(-1)
checks.append(pos+headIFrameTS-ts1) checks.append(pos+head_iframe_ts-ts1)
subparts = [] subparts = []
@@ -3311,26 +3312,26 @@ def main():
# if args.coarse: # if args.coarse:
# doCoarseProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], inputFile=mkv, # doCoarseProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], inputFile=mkv,
# begin=ts1, end=headIFrameTS, nbFrames=nbHeadFrames-1, # begin=ts1, end=head_iframe_ts, nbFrames=nb_head_frames-1,
# frameRate=frameRate, filesPrefix='part-%d-head' % (partnum), # frameRate=frameRate, filesPrefix='part-%d-head' % (partnum),
# streams=streams, width=width, height=height, # streams=streams, width=width, height=height,
# temporaries=temporaries, dumpMemFD=args.dump) # temporaries=temporaries, dumpMemFD=args.dump)
# else: # else:
# doFineGrainProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], # doFineGrainProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'],
# inputFile=mkv, begin=ts1, end=headIFrameTS, # inputFile=mkv, begin=ts1, end=head_iframe_ts,
# nbFrames=nbHeadFrames-1, frameRate=frameRate, # nbFrames=nb_head_frames-1, frameRate=frameRate,
# filesPrefix='part-%d-head' % (partnum), streams=streams, # filesPrefix='part-%d-head' % (partnum), streams=streams,
# width=width, height=height, temporaries=temporaries, # width=width, height=height, temporaries=temporaries,
# dumpMemFD=args.dump) # dumpMemFD=args.dump)
if (not args.coarse) and (nbHeadFrames > args.threshold): if (not args.coarse) and (nb_head_frames > args.threshold):
# We extract all frames between the beginning upto the frame that immediately preceeds # We extract all frames between the beginning upto the frame that immediately preceeds
# the I-frame. # the I-frame.
h264Head, h264HeadTS, mkvHead = extractAllStreams(ffmpeg=paths['ffmpeg'], h264_head, h264_head_ts, mkv_head = extractAllStreams(ffmpeg=paths['ffmpeg'],
ffprobe=paths['ffprobe'], ffprobe=paths['ffprobe'],
inputFile=mkv, begin=ts1, inputFile=mkv, begin=ts1,
end=headIFrameTS, end=head_iframe_ts,
nbFrames=nbHeadFrames-1, nbFrames=nb_head_frames-1,
framerate=framerate, framerate=framerate,
filesPrefix=f'part-{partnum:d}-head', filesPrefix=f'part-{partnum:d}-head',
streams=streams, width=width, streams=streams, width=width,
@@ -3339,14 +3340,14 @@ def main():
dumpMemFD=args.dump) dumpMemFD=args.dump)
# If we are not at an exact boundary: # If we are not at an exact boundary:
if mkvHead is not None: if mkv_head is not None:
subparts.append(mkvHead) subparts.append(mkv_head)
if h264Head is not None: if h264_head is not None:
avcconfig = getAvcConfigFromH264(h264Head) avcconfig = getAvcConfigFromH264(h264_head)
otherAvcConfigs.append(avcconfig) other_avc_configs.append(avcconfig)
h264parts.append(h264Head) h264parts.append(h264_head)
if h264HeadTS is not None: if h264_head_ts is not None:
h264TS.append(h264HeadTS) h264_ts.append(h264_head_ts)
# Creating MKV file that corresponds to current part between I-frames # Creating MKV file that corresponds to current part between I-frames
# Internal video with all streams (video, audio and subtitles) # Internal video with all streams (video, audio and subtitles)
@@ -3359,25 +3360,25 @@ def main():
internalNoVideoMKVName = f'part-{partnum:d}-internal-novideo.mkv' internalNoVideoMKVName = f'part-{partnum:d}-internal-novideo.mkv'
try: try:
internalMKV = open(internalMKVName, 'w+') internalMKV = open(internalMKVName, 'w+', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file: %s', internalMKVName) logger.error('Impossible to create file: %s', internalMKVName)
exit(-1) exit(-1)
try: try:
internalNoVideoMKV = open(internalNoVideoMKVName, 'w+') internalNoVideoMKV = open(internalNoVideoMKVName, 'w+', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file: %s', internalNoVideoMKVName) logger.error('Impossible to create file: %s', internalNoVideoMKVName)
exit(-1) exit(-1)
try: try:
internalH264 = open(internalH264Name, 'w+') internalH264 = open(internalH264Name, 'w+', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file: %s', internalH264Name) logger.error('Impossible to create file: %s', internalH264Name)
exit(-1) exit(-1)
try: try:
internalH264TS = open(internalH264TSName, 'w+') internalH264TS = open(internalH264TSName, 'w+', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file: %s', internalH264TSName) logger.error('Impossible to create file: %s', internalH264TSName)
exit(-1) exit(-1)
@@ -3385,7 +3386,7 @@ def main():
# logger.info('Merge header, middle and trailer subpart into: %s' % internalMKVName) # logger.info('Merge header, middle and trailer subpart into: %s' % internalMKVName)
# Extract internal part of MKV # Extract internal part of MKV
extractMKVPart(mkvmerge=paths['mkvmerge'], inputFile=mkv, outputFile=internalMKV, extractMKVPart(mkvmerge=paths['mkvmerge'], inputFile=mkv, outputFile=internalMKV,
begin=headIFrameTS, end=tailIFrameTS) begin=head_iframe_ts, end=tail_iframe_ts)
# Extract video stream of internal part as a raw H264 and its timestamps. # Extract video stream of internal part as a raw H264 and its timestamps.
logger.info('Extract video track as raw H264 file.') logger.info('Extract video track as raw H264 file.')
@@ -3403,15 +3404,15 @@ def main():
temporaries.append(internalNoVideoMKV) temporaries.append(internalNoVideoMKV)
h264parts.append(internalH264) h264parts.append(internalH264)
h264TS.append(internalH264TS) h264_ts.append(internalH264TS)
subparts.append(internalNoVideoMKV) subparts.append(internalNoVideoMKV)
if (not args.coarse) and (nbTailFrames > args.threshold): if (not args.coarse) and (nb_tail_frames > args.threshold):
# We extract all frames between the I-frame (including it) upto the end. # We extract all frames between the I-frame (including it) upto the end.
h264Tail, h264TailTS, mkvTail = extractAllStreams(ffmpeg=paths['ffmpeg'], h264Tail, h264TailTS, mkvTail = extractAllStreams(ffmpeg=paths['ffmpeg'],
ffprobe=paths['ffprobe'], ffprobe=paths['ffprobe'],
inputFile=mkv, begin=tailIFrameTS, inputFile=mkv, begin=tail_iframe_ts,
end=ts2, nbFrames=nbTailFrames, end=ts2, nbFrames=nb_tail_frames,
framerate=framerate, framerate=framerate,
filesPrefix=f'part-{partnum:d}-tail', filesPrefix=f'part-{partnum:d}-tail',
streams=streams, streams=streams,
@@ -3423,10 +3424,10 @@ def main():
subparts.append(mkvTail) subparts.append(mkvTail)
if h264Tail is not None: if h264Tail is not None:
avcconfig = getAvcConfigFromH264(h264Tail) avcconfig = getAvcConfigFromH264(h264Tail)
otherAvcConfigs.append(avcconfig) other_avc_configs.append(avcconfig)
h264parts.append(h264Tail) h264parts.append(h264Tail)
if h264TailTS is not None: if h264TailTS is not None:
h264TS.append(h264TailTS) h264_ts.append(h264TailTS)
logger.info('Merging MKV: %s', subparts) logger.info('Merging MKV: %s', subparts)
part = mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=subparts, part = mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=subparts,
@@ -3434,20 +3435,20 @@ def main():
mkvparts.append(part) mkvparts.append(part)
temporaries.append(part) temporaries.append(part)
pos = pos+tailIFrameTS-ts1 pos = pos+tail_iframe_ts-ts1
# We need to check the end also # We need to check the end also
checks.append(pos) checks.append(pos)
# When using coarse option there is a single AVC configuration. # When using coarse option there is a single AVC configuration.
for avcConfig in otherAvcConfigs: for avcConfig in other_avc_configs:
main_avc_config.merge(avcConfig) main_avc_config.merge(avcConfig)
logger.debug('Merged AVC configuration: %s', main_avc_config) logger.debug('Merged AVC configuration: %s', main_avc_config)
nbMKVParts = len(mkvparts) nbMKVParts = len(mkvparts)
if nbMKVParts > 0: if nbMKVParts > 0:
try: try:
fullH264 = open(f'{basename}-full.h264', 'w+') fullH264 = open(f'{basename}-full.h264', 'w+', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file full H264 stream.') logger.error('Impossible to create file full H264 stream.')
exit(-1) exit(-1)
@@ -3457,13 +3458,13 @@ def main():
temporaries.append(fullH264) temporaries.append(fullH264)
try: try:
fullH264TS = open(f'{basename}-ts.txt', 'w+') fullH264TS = open(f'{basename}-ts.txt', 'w+', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to create file containing all video timestamps.') logger.error('Impossible to create file containing all video timestamps.')
exit(-1) exit(-1)
logger.info('Merging H264 timestamps') logger.info('Merging H264 timestamps')
concatenateH264TSParts(h264TSParts=h264TS, output=fullH264TS) concatenateH264TSParts(h264TSParts=h264_ts, output=fullH264TS)
temporaries.append(fullH264TS) temporaries.append(fullH264TS)
finalNoVideoName = f'{basename}-novideo.mkv' finalNoVideoName = f'{basename}-novideo.mkv'
@@ -3481,7 +3482,7 @@ def main():
if nbMKVParts >=1 : if nbMKVParts >=1 :
try: try:
finalNoVideo = open(finalNoVideoName, 'r') finalNoVideo = open(finalNoVideoName, 'r', encoding='utf8')
except IOError: except IOError:
logger.error('Impossible to open file: %s.', finalNoVideoName) logger.error('Impossible to open file: %s.', finalNoVideoName)
exit(-1) exit(-1)
@@ -3536,12 +3537,12 @@ def main():
logger.info(listOfSubtitles) logger.info(listOfSubtitles)
for idx_name, sub_name, _, _ in listOfSubtitles: for idx_name, sub_name, _, _ in listOfSubtitles:
try: try:
idx = open(idx_name,'r') idx = open(idx_name,'r', encoding='utf8')
except IOError: except IOError:
logger.error("Impossible to open %s.", idx_name) logger.error("Impossible to open %s.", idx_name)
exit(-1) exit(-1)
try: try:
sub = open(sub_name,'r') sub = open(sub_name,'r', encoding='utf8')
except IOError: except IOError:
logger.error("Impossible to open %s.", sub_name) logger.error("Impossible to open %s.", sub_name)
exit(-1) exit(-1)