diff --git a/removeads.py b/removeads.py index d1011af..3b9b690 100755 --- a/removeads.py +++ b/removeads.py @@ -294,20 +294,19 @@ def doOCR(vobsubocr, idxs, duration, temporaries, dumpMemFD=False): if dumpMemFD: try: - dump_srt = open(srtname,'w') + with open(srtname,'w', encoding='utf8') as dump_srt: + lseek(srtfd, 0, SEEK_SET) + srt_length = fstat(srtfd).st_size + buf = read(srtfd, srt_length) + outfd = dump_srt.fileno() + pos = 0 + while pos < srt_length: + pos+=write(outfd, buf[pos:]) + temporaries.append(dump_srt) except IOError: logger.error('Impossible to create file: %s', srtname) return None - lseek(srtfd, 0, SEEK_SET) - srt_length = fstat(srtfd).st_size - buf = read(srtfd, srt_length) - outfd = dump_srt.fileno() - pos = 0 - while pos < srt_length: - pos+=write(outfd, buf[pos:]) - - temporaries.append(dump_srt) srt_length = fstat(srtfd).st_size if srt_length > 0: @@ -2471,7 +2470,7 @@ def dumpPPM(pictures, prefix, temporaries): header = BytesIO(pictures[pos:]) magic = header.readline().decode('utf8') dimensions = header.readline().decode('utf8') - max_value = header.readline().decode('utf8') + max_value = int(header.readline().decode('utf8')) if magic == 'P6\n': pattern = re.compile('^(?P[0-9]+) (?P[0-9]+)\n$') m = pattern.match(dimensions) @@ -2491,7 +2490,7 @@ def dumpPPM(pictures, prefix, temporaries): header_len=2+1+ceil(log(width, 10))+1+ceil(log(height, 10))+1+3+1 try: - out = open(filename, 'w') + out = open(filename, 'w', encoding='utf8') outfd = out.fileno() except IOError: logger.error('Impossible to create file: %s', filename) @@ -2630,7 +2629,7 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref if dumpMemFD: try: - output = open(tmpname,'w') + output = open(tmpname,'w', encoding='utf8') except IOError: logger.error('Impossible to create file: %s', tmpname) return None @@ -2722,7 +2721,7 @@ def extractAllStreams(ffmpeg, ffprobe, inputFile, begin, end, streams, filesPref h264_ts_filename = f'{filesPrefix}-ts.txt' try: - h264_ts_output = open(h264_ts_filename,'w+') + h264_ts_output = open(h264_ts_filename,'w+', encoding='utf8') except IOError: logger.error('Impossible to create file: %s', h264_ts_filename) return None @@ -2752,7 +2751,7 @@ def mergeMKVs(mkvmerge, inputs, outputName, concatenate=True, timestamps=None): fds = [] try: - out = open(outputName, 'w+') + out = open(outputName, 'w+', encoding='utf8') except IOError: logger.error('Impossible to create file: %s', outputName) return None @@ -2770,7 +2769,7 @@ def mergeMKVs(mkvmerge, inputs, outputName, concatenate=True, timestamps=None): first = True partnum = 0 for mkv in inputs: - if mkv !=None: + if mkv is not None: fd = mkv.fileno() fds.append(fd) set_inheritable(fd, True) @@ -2929,7 +2928,7 @@ def remuxSRTSubtitles(mkvmerge, inputFile, outputFileName, subtitles): logger = logging.getLogger(__name__) try: - out = open(outputFileName, 'w') + out = open(outputFileName, 'w', encoding='utf8') except IOError: logger.error('Impossible to create file: %s', outputFileName) return None @@ -3039,7 +3038,7 @@ def doCoarseProcessing(ffmpeg, ffprobe, mkvmerge, inputFile, begin, end, nbFrame internal_mkv_name = f'{filesPrefix}.mkv' try: - internal_mkv = open(internal_mkv_name, 'w+') + internal_mkv = open(internal_mkv_name, 'w+', encoding='utf8') except IOError: logger.error('Impossible to create file: %s', internal_mkv_name) exit(-1) @@ -3133,7 +3132,7 @@ def main(): mkvfilename = basename+'.mkv' try: - input_file = open(args.input_file, mode='r') + input_file = open(args.input_file, mode='r', encoding='utf8') except IOError: logger.error("Impossible to open %s", args.input_file) exit(-1) @@ -3170,13 +3169,13 @@ def main(): if format_of_file == SupportedFormat.TS: logger.info("Converting TS to MP4 (to fix timestamps).") try: - with open(mp4filename, 'w+') as mp4: + with open(mp4filename, 'w+', encoding='utf8') as mp4: ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], input_file, 'mpegts', mp4, 'mp4', duration) temporaries.append(mp4) logger.info("Converting MP4 to MKV.") try: - mkv = open(mkvfilename, 'w+') + mkv = open(mkvfilename, 'w+', encoding='utf8') except IOError: logger.error('') @@ -3190,7 +3189,7 @@ def main(): elif format_of_file == SupportedFormat.MP4: logger.info("Converting MP4 to MKV") try: - mkv = open(mkvfilename, 'w+') + mkv = open(mkvfilename, 'w+', encoding='utf8') except IOError: logger.error('') ffmpegConvert(paths['ffmpeg'], paths['ffprobe'], input_file, 'mp4', mkv, 'matroska', @@ -3235,31 +3234,31 @@ def main(): logger.debug('AVC configuration: %s', main_avc_config) # We check if the parse and dump operations are idempotent. - privateData = dumpCodecPrivateData(main_avc_config) - logger.debug('Redump AVC configuration: %s', hexdump.dump(privateData, sep=':')) + private_data = dumpCodecPrivateData(main_avc_config) + logger.debug('Redump AVC configuration: %s', hexdump.dump(private_data, sep=':')) # In rare occasion, the PPS has trailing zeroes that do not seem to be related to useful data # but they differ from the private data we generate that do not contain them. # In that case we try to redecode our own private data to see if both AVC configurations are # the same. - if main_codec_private_data != privateData: + if main_codec_private_data != private_data: logger.warning('Difference detected in bitstream !!') - isoAvcConfig = parseCodecPrivate(privateData) - logger.debug('Reread AVC configuration: %s', isoAvcConfig) + iso_avc_config = parseCodecPrivate(private_data) + logger.debug('Reread AVC configuration: %s', iso_avc_config) # If there exists a difference between our own reconstructed AVC configuration and the # original one, we abandon - if isoAvcConfig != main_avc_config: - logger.error('AVC configurations are different: %s\n%s\n', main_avc_config, isoAvcConfig) + if iso_avc_config != main_avc_config: + logger.error('AVC configurations are different: %s\n%s\n', main_avc_config, iso_avc_config) exit(-1) # Pour chaque portion partnum = 0 mkvparts = [] h264parts = [] - h264TS = [] + h264_ts = [] checks = [] pos = timedelta() - otherAvcConfigs = [] + other_avc_configs = [] for ts1, ts2 in parts: # TODO: translate comment in english @@ -3278,32 +3277,34 @@ def main(): partnum = partnum + 1 # Get the nearest I-frame whose timestamp is greater or equal to the beginning. - headFrames = getNearestIFrame(paths['ffprobe'], mkv, ts1, before=False) - if headFrames is None: + head_frames = getNearestIFrame(paths['ffprobe'], mkv, ts1, before=False) + if head_frames is None: + logger.error('Impossible to retrieve I-frame') exit(-1) # Get the nearest I-frame whose timestamp ... # TODO: wrong here ... - tailFrames = getNearestIFrame(paths['ffprobe'], mkv, ts2, before=True) - if tailFrames is None: + tail_frames = getNearestIFrame(paths['ffprobe'], mkv, ts2, before=True) + if tail_frames is None: + logger.error('Impossible to retrieve I-frame') exit(-1) - nbHeadFrames, headIFrame = headFrames - nbTailFrames, tailIFrame = tailFrames + nb_head_frames, head_iframe = head_frames + nb_tail_frames, tail_iframe = tail_frames logger.info("Found %d frames between beginning of current part and first I-frame", - nbHeadFrames) + nb_head_frames) logger.info("Found %d frames between last I-frame and end of current part", - nbTailFrames) + nb_tail_frames) - headIFrameTS = getTSFrame(headIFrame) - if headIFrameTS is None: + head_iframe_ts = getTSFrame(head_iframe) + if head_iframe_ts is None: exit(-1) - tailIFrameTS = getTSFrame(tailIFrame) - if tailIFrameTS is None: + tail_iframe_ts = getTSFrame(tail_iframe) + if tail_iframe_ts is None: exit(-1) - checks.append(pos+headIFrameTS-ts1) + checks.append(pos+head_iframe_ts-ts1) subparts = [] @@ -3311,26 +3312,26 @@ def main(): # if args.coarse: # doCoarseProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], inputFile=mkv, - # begin=ts1, end=headIFrameTS, nbFrames=nbHeadFrames-1, + # begin=ts1, end=head_iframe_ts, nbFrames=nb_head_frames-1, # frameRate=frameRate, filesPrefix='part-%d-head' % (partnum), # streams=streams, width=width, height=height, # temporaries=temporaries, dumpMemFD=args.dump) # else: # doFineGrainProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], - # inputFile=mkv, begin=ts1, end=headIFrameTS, - # nbFrames=nbHeadFrames-1, frameRate=frameRate, + # inputFile=mkv, begin=ts1, end=head_iframe_ts, + # nbFrames=nb_head_frames-1, frameRate=frameRate, # filesPrefix='part-%d-head' % (partnum), streams=streams, # width=width, height=height, temporaries=temporaries, # dumpMemFD=args.dump) - if (not args.coarse) and (nbHeadFrames > args.threshold): + if (not args.coarse) and (nb_head_frames > args.threshold): # We extract all frames between the beginning upto the frame that immediately preceeds # the I-frame. - h264Head, h264HeadTS, mkvHead = extractAllStreams(ffmpeg=paths['ffmpeg'], + h264_head, h264_head_ts, mkv_head = extractAllStreams(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], inputFile=mkv, begin=ts1, - end=headIFrameTS, - nbFrames=nbHeadFrames-1, + end=head_iframe_ts, + nbFrames=nb_head_frames-1, framerate=framerate, filesPrefix=f'part-{partnum:d}-head', streams=streams, width=width, @@ -3339,14 +3340,14 @@ def main(): dumpMemFD=args.dump) # If we are not at an exact boundary: - if mkvHead is not None: - subparts.append(mkvHead) - if h264Head is not None: - avcconfig = getAvcConfigFromH264(h264Head) - otherAvcConfigs.append(avcconfig) - h264parts.append(h264Head) - if h264HeadTS is not None: - h264TS.append(h264HeadTS) + if mkv_head is not None: + subparts.append(mkv_head) + if h264_head is not None: + avcconfig = getAvcConfigFromH264(h264_head) + other_avc_configs.append(avcconfig) + h264parts.append(h264_head) + if h264_head_ts is not None: + h264_ts.append(h264_head_ts) # Creating MKV file that corresponds to current part between I-frames # Internal video with all streams (video, audio and subtitles) @@ -3359,25 +3360,25 @@ def main(): internalNoVideoMKVName = f'part-{partnum:d}-internal-novideo.mkv' try: - internalMKV = open(internalMKVName, 'w+') + internalMKV = open(internalMKVName, 'w+', encoding='utf8') except IOError: logger.error('Impossible to create file: %s', internalMKVName) exit(-1) try: - internalNoVideoMKV = open(internalNoVideoMKVName, 'w+') + internalNoVideoMKV = open(internalNoVideoMKVName, 'w+', encoding='utf8') except IOError: logger.error('Impossible to create file: %s', internalNoVideoMKVName) exit(-1) try: - internalH264 = open(internalH264Name, 'w+') + internalH264 = open(internalH264Name, 'w+', encoding='utf8') except IOError: logger.error('Impossible to create file: %s', internalH264Name) exit(-1) try: - internalH264TS = open(internalH264TSName, 'w+') + internalH264TS = open(internalH264TSName, 'w+', encoding='utf8') except IOError: logger.error('Impossible to create file: %s', internalH264TSName) exit(-1) @@ -3385,7 +3386,7 @@ def main(): # logger.info('Merge header, middle and trailer subpart into: %s' % internalMKVName) # Extract internal part of MKV extractMKVPart(mkvmerge=paths['mkvmerge'], inputFile=mkv, outputFile=internalMKV, - begin=headIFrameTS, end=tailIFrameTS) + begin=head_iframe_ts, end=tail_iframe_ts) # Extract video stream of internal part as a raw H264 and its timestamps. logger.info('Extract video track as raw H264 file.') @@ -3403,15 +3404,15 @@ def main(): temporaries.append(internalNoVideoMKV) h264parts.append(internalH264) - h264TS.append(internalH264TS) + h264_ts.append(internalH264TS) subparts.append(internalNoVideoMKV) - if (not args.coarse) and (nbTailFrames > args.threshold): + if (not args.coarse) and (nb_tail_frames > args.threshold): # We extract all frames between the I-frame (including it) upto the end. h264Tail, h264TailTS, mkvTail = extractAllStreams(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], - inputFile=mkv, begin=tailIFrameTS, - end=ts2, nbFrames=nbTailFrames, + inputFile=mkv, begin=tail_iframe_ts, + end=ts2, nbFrames=nb_tail_frames, framerate=framerate, filesPrefix=f'part-{partnum:d}-tail', streams=streams, @@ -3423,10 +3424,10 @@ def main(): subparts.append(mkvTail) if h264Tail is not None: avcconfig = getAvcConfigFromH264(h264Tail) - otherAvcConfigs.append(avcconfig) + other_avc_configs.append(avcconfig) h264parts.append(h264Tail) if h264TailTS is not None: - h264TS.append(h264TailTS) + h264_ts.append(h264TailTS) logger.info('Merging MKV: %s', subparts) part = mergeMKVs(mkvmerge=paths['mkvmerge'], inputs=subparts, @@ -3434,20 +3435,20 @@ def main(): mkvparts.append(part) temporaries.append(part) - pos = pos+tailIFrameTS-ts1 + pos = pos+tail_iframe_ts-ts1 # We need to check the end also checks.append(pos) # When using coarse option there is a single AVC configuration. - for avcConfig in otherAvcConfigs: + for avcConfig in other_avc_configs: main_avc_config.merge(avcConfig) logger.debug('Merged AVC configuration: %s', main_avc_config) nbMKVParts = len(mkvparts) if nbMKVParts > 0: try: - fullH264 = open(f'{basename}-full.h264', 'w+') + fullH264 = open(f'{basename}-full.h264', 'w+', encoding='utf8') except IOError: logger.error('Impossible to create file full H264 stream.') exit(-1) @@ -3457,13 +3458,13 @@ def main(): temporaries.append(fullH264) try: - fullH264TS = open(f'{basename}-ts.txt', 'w+') + fullH264TS = open(f'{basename}-ts.txt', 'w+', encoding='utf8') except IOError: logger.error('Impossible to create file containing all video timestamps.') exit(-1) logger.info('Merging H264 timestamps') - concatenateH264TSParts(h264TSParts=h264TS, output=fullH264TS) + concatenateH264TSParts(h264TSParts=h264_ts, output=fullH264TS) temporaries.append(fullH264TS) finalNoVideoName = f'{basename}-novideo.mkv' @@ -3481,7 +3482,7 @@ def main(): if nbMKVParts >=1 : try: - finalNoVideo = open(finalNoVideoName, 'r') + finalNoVideo = open(finalNoVideoName, 'r', encoding='utf8') except IOError: logger.error('Impossible to open file: %s.', finalNoVideoName) exit(-1) @@ -3536,12 +3537,12 @@ def main(): logger.info(listOfSubtitles) for idx_name, sub_name, _, _ in listOfSubtitles: try: - idx = open(idx_name,'r') + idx = open(idx_name,'r', encoding='utf8') except IOError: logger.error("Impossible to open %s.", idx_name) exit(-1) try: - sub = open(sub_name,'r') + sub = open(sub_name,'r', encoding='utf8') except IOError: logger.error("Impossible to open %s.", sub_name) exit(-1)