Compare commits
25
Commits
294d857206
...
refactor
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
814c64cc52 | ||
|
|
c17ccb6563 | ||
|
|
e1d7474e00 | ||
|
|
51bab76c00 | ||
|
|
153873637e | ||
|
|
b0bc0a338e | ||
|
|
4597c459a8 | ||
|
|
5b2afd29b3 | ||
|
|
d1df44c82c | ||
|
|
1a616f6cbd | ||
|
|
d756fede54 | ||
|
|
34d1bc26f3 | ||
|
|
04781021eb | ||
|
|
e1ac3e48bb | ||
|
|
ae1a040590 | ||
|
|
0345293664 | ||
|
|
63d2deafb1 | ||
|
|
6f5dd3a9e4 | ||
|
|
093ad733d3 | ||
|
|
a6a41c112e | ||
|
|
447d04d13d | ||
|
|
2590faf20f | ||
|
|
bea482487e | ||
|
|
c48b77f294 | ||
|
|
0ff2e3a6dd |
+32
-491
@@ -6,46 +6,18 @@
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
import os.path
|
from datetime import timedelta
|
||||||
from datetime import datetime, timedelta
|
|
||||||
from os import unlink
|
|
||||||
from shutil import copyfile, move
|
|
||||||
from sys import exit
|
|
||||||
from functools import cmp_to_key
|
from functools import cmp_to_key
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
import coloredlogs
|
import coloredlogs
|
||||||
import hexdump
|
|
||||||
|
|
||||||
from .tscut import (
|
from tscut.exceptions import TSCutError
|
||||||
SupportedFormat,
|
from tscut.models import ProcessingOptions
|
||||||
change_codec_private_data,
|
from tscut.pipeline import process_recording
|
||||||
check_required_tools,
|
from tscut.temporaries import TemporaryFiles
|
||||||
compare_time_interval,
|
from tscut.tools.discovery import check_required_tools
|
||||||
concatenate_h264_parts,
|
from tscut.tools.timeframe import compare_time_interval, parse_time_interval
|
||||||
concatenate_h264_ts_parts,
|
|
||||||
do_ocr,
|
|
||||||
dump_codec_private_data,
|
|
||||||
extract_all_streams,
|
|
||||||
extract_mkv_part,
|
|
||||||
extract_srt,
|
|
||||||
extract_track_from_mkv,
|
|
||||||
ffmpeg_convert,
|
|
||||||
find_subtitles_tracks,
|
|
||||||
get_avc_config_from_h264,
|
|
||||||
get_codec_private_data_from_mkv,
|
|
||||||
get_format,
|
|
||||||
get_frame_rate,
|
|
||||||
get_movie_duration,
|
|
||||||
get_nearest_iframe,
|
|
||||||
get_streams,
|
|
||||||
get_tesseract_supported_lang,
|
|
||||||
get_ts_frame,
|
|
||||||
merge_mkvs,
|
|
||||||
parse_codec_private,
|
|
||||||
parse_time_interval,
|
|
||||||
remove_video_tracks_from_mkv,
|
|
||||||
remux_srt_subtitles,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -88,7 +60,7 @@ def main() -> int:
|
|||||||
|
|
||||||
if args.coarse and args.threshold is not None:
|
if args.coarse and args.threshold is not None:
|
||||||
logger.error('--coarse and threshold arguments are exclusive.')
|
logger.error('--coarse and threshold arguments are exclusive.')
|
||||||
exit(-1)
|
return 2
|
||||||
|
|
||||||
if (not args.coarse) and args.threshold is None:
|
if (not args.coarse) and args.threshold is None:
|
||||||
args.threshold = 0
|
args.threshold = 0
|
||||||
@@ -108,7 +80,7 @@ def main() -> int:
|
|||||||
ts1, ts2 = parse_time_interval(interval)
|
ts1, ts2 = parse_time_interval(interval)
|
||||||
if ts1 is None or ts2 is None:
|
if ts1 is None or ts2 is None:
|
||||||
logger.error("Illegal time interval: %s", interval)
|
logger.error("Illegal time interval: %s", interval)
|
||||||
exit(-1)
|
return 2
|
||||||
parts.append((ts1,ts2))
|
parts.append((ts1,ts2))
|
||||||
|
|
||||||
# Sort intervals
|
# Sort intervals
|
||||||
@@ -120,463 +92,32 @@ def main() -> int:
|
|||||||
ts1, ts2 = part
|
ts1, ts2 = part
|
||||||
if prevts > ts1:
|
if prevts > ts1:
|
||||||
logger.error('Intervals are overlapping')
|
logger.error('Intervals are overlapping')
|
||||||
exit(-1)
|
return 2
|
||||||
prevts = ts2
|
prevts = ts2
|
||||||
|
|
||||||
nb_parts = len(parts)
|
config = ProcessingOptions(
|
||||||
temporaries = []
|
input_file=Path(args.input_file),
|
||||||
|
output_file=Path(args.output_file),
|
||||||
basename = os.path.splitext(os.path.basename(args.input_file))[0]
|
parts=parts,
|
||||||
mp4filename = basename+'.mp4'
|
framerate = args.framerate,
|
||||||
mkvfilename = basename+'.mkv'
|
verbose=args.verbose,
|
||||||
|
coarse=args.coarse,
|
||||||
|
threshold=args.threshold,
|
||||||
|
keep_temporaries=args.keep,
|
||||||
|
subtitles_ocr=args.srt,
|
||||||
|
dump_memory=args.dump,
|
||||||
|
tools_paths = paths,
|
||||||
|
all_optional_tools = all_optional_tools)
|
||||||
|
|
||||||
|
logger.debug("Configuration: %s", config)
|
||||||
|
temporaries = TemporaryFiles(config.keep_temporaries)
|
||||||
try:
|
try:
|
||||||
input_file = open(args.input_file, mode='rb')
|
process_recording(config, temporaries)
|
||||||
logger.debug("Type of input file: %s", type(input_file))
|
except TSCutError as exc:
|
||||||
except OSError:
|
logger.error("%s", exc)
|
||||||
logger.error("Impossible to open %s", args.input_file)
|
return 1
|
||||||
exit(-1)
|
finally:
|
||||||
|
|
||||||
format_of_file = get_format(paths['ffprobe'], input_file)
|
|
||||||
|
|
||||||
if format_of_file is None:
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
duration = timedelta(seconds=float(format_of_file['duration']))
|
|
||||||
logger.info("Durée de l'enregistrement: %s", duration)
|
|
||||||
|
|
||||||
if args.framerate is None:
|
|
||||||
framerate = get_frame_rate(paths['ffprobe'], input_file)
|
|
||||||
if framerate is None:
|
|
||||||
logger.error('Impossible to estimate frame rate !')
|
|
||||||
exit(-1)
|
|
||||||
else:
|
|
||||||
framerate = args.framerate
|
|
||||||
|
|
||||||
logger.info('Frame rate: %.1f fps', framerate)
|
|
||||||
|
|
||||||
found = False
|
|
||||||
for f in SupportedFormat:
|
|
||||||
if 'format_name' in format_of_file:
|
|
||||||
if format_of_file['format_name'] == str(f):
|
|
||||||
found = True
|
|
||||||
format_of_file = f
|
|
||||||
break
|
|
||||||
|
|
||||||
if not found:
|
|
||||||
logger.error('Unsupported format of file')
|
|
||||||
|
|
||||||
if format_of_file == SupportedFormat.TS:
|
|
||||||
logger.info("Converting TS to MP4 (to fix timestamps).")
|
|
||||||
try:
|
|
||||||
with open(mp4filename, 'wb+') as mp4:
|
|
||||||
ffmpeg_convert(paths['ffmpeg'], paths['ffprobe'], input_file, 'mpegts', mp4, 'mp4',
|
|
||||||
duration)
|
|
||||||
temporaries.append(mp4)
|
|
||||||
logger.info("Converting MP4 to MKV.")
|
|
||||||
try:
|
|
||||||
mkv = open(mkvfilename, 'wb+')
|
|
||||||
except OSError:
|
|
||||||
logger.error('')
|
|
||||||
|
|
||||||
ffmpeg_convert(paths['ffmpeg'], paths['ffprobe'], mp4, 'mp4', mkv, 'matroska',
|
|
||||||
duration)
|
|
||||||
if nb_parts > 0:
|
|
||||||
temporaries.append(mkv)
|
|
||||||
except OSError:
|
|
||||||
logger.error('')
|
|
||||||
|
|
||||||
elif format_of_file == SupportedFormat.MP4:
|
|
||||||
logger.info("Converting MP4 to MKV")
|
|
||||||
try:
|
|
||||||
mkv = open(mkvfilename, 'wb+')
|
|
||||||
except OSError:
|
|
||||||
logger.error('')
|
|
||||||
ffmpeg_convert(paths['ffmpeg'], paths['ffprobe'], input_file, 'mp4', mkv, 'matroska',
|
|
||||||
duration)
|
|
||||||
if nb_parts > 0:
|
|
||||||
temporaries.append(mkv)
|
|
||||||
else:
|
|
||||||
logger.info("Already in MKV")
|
|
||||||
mkv = input_file
|
|
||||||
|
|
||||||
streams = get_streams(paths['ffprobe'], mkv)
|
|
||||||
|
|
||||||
logger.debug('Streams: %s', streams)
|
|
||||||
main_video = None
|
|
||||||
nb_videos = 0
|
|
||||||
for stream in streams:
|
|
||||||
if stream['codec_type'] == 'video':
|
|
||||||
if stream['disposition']['default'] == 1:
|
|
||||||
main_video = stream
|
|
||||||
width = stream['width']
|
|
||||||
height = stream['height']
|
|
||||||
break
|
|
||||||
nb_videos+=1
|
|
||||||
if nb_videos == 1:
|
|
||||||
main_video = stream
|
|
||||||
width = stream['width']
|
|
||||||
height = stream['height']
|
|
||||||
else:
|
|
||||||
main_video = None
|
|
||||||
|
|
||||||
if main_video is None:
|
|
||||||
logger.error('Impossible to find main video stream.')
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
# We retrieve the main private codec data
|
|
||||||
_, main_codec_private_data = get_codec_private_data_from_mkv(mkvinfo_path=paths['mkvinfo'],
|
|
||||||
input_file=mkv)
|
|
||||||
logger.debug('Main video stream has following private data: %s',
|
|
||||||
hexdump.dump(main_codec_private_data, sep=':'))
|
|
||||||
|
|
||||||
# We parse them
|
|
||||||
main_avc_config = parse_codec_private(main_codec_private_data)
|
|
||||||
logger.debug('AVC configuration: %s', main_avc_config)
|
|
||||||
|
|
||||||
# We check if the parse and dump operations are idempotent.
|
|
||||||
private_data = dump_codec_private_data(main_avc_config)
|
|
||||||
logger.debug('Redump AVC configuration: %s', hexdump.dump(private_data, sep=':'))
|
|
||||||
# In rare occasion, the PPS has trailing zeroes that do not seem to be related to useful data
|
|
||||||
# but they differ from the private data we generate that do not contain them.
|
|
||||||
# In that case we try to redecode our own private data to see if both AVC configurations are
|
|
||||||
# the same.
|
|
||||||
if main_codec_private_data != private_data:
|
|
||||||
logger.warning('Difference detected in bitstream !!')
|
|
||||||
iso_avc_config = parse_codec_private(private_data)
|
|
||||||
logger.debug('Reread AVC configuration: %s', iso_avc_config)
|
|
||||||
# If there exists a difference between our own reconstructed AVC configuration and the
|
|
||||||
# original one, we abandon
|
|
||||||
if iso_avc_config != main_avc_config:
|
|
||||||
logger.error('AVC configurations are different: %s\n%s\n', main_avc_config,
|
|
||||||
iso_avc_config)
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
# Pour chaque portion
|
|
||||||
partnum = 0
|
|
||||||
mkvparts = []
|
|
||||||
h264parts = []
|
|
||||||
h264_ts = []
|
|
||||||
checks = []
|
|
||||||
pos = timedelta()
|
|
||||||
|
|
||||||
other_avc_configs = []
|
|
||||||
|
|
||||||
for ts1, ts2 in parts:
|
|
||||||
# TODO: translate comment in english
|
|
||||||
# Trouver l'estampille de la trame 'I' la plus proche (mais postérieure) au début
|
|
||||||
# de la portion.
|
|
||||||
# Trouver l'estampille de la trame 'I' la plus proche (mais antérieure) à la fin
|
|
||||||
# de la portion.
|
|
||||||
# On a alors
|
|
||||||
# debut ----- trame --------- trame --------- fin fin+1
|
|
||||||
# 'B/P' 'B/P'* 'I' 'I' 'B/P'* 'B/P' 'I/B/P'
|
|
||||||
# Si la trame de début est déjà 'I', il n'y a rien à faire.
|
|
||||||
# Sinon on extrait les trames 'B' ou 'P' depuis le début jusqu'à la trame 'I' non incluse.
|
|
||||||
# Si la trame de fin précède une trame I, on n'a rien à faire.
|
|
||||||
# Sinon on extrait toutes les trames depuis la dernière trame I jusqu'à la trame de fin.
|
|
||||||
|
|
||||||
partnum = partnum + 1
|
|
||||||
|
|
||||||
# Get the nearest I-frame whose timestamp is greater or equal to the beginning.
|
|
||||||
head_frames = get_nearest_iframe(paths['ffprobe'], mkv, ts1, before=False)
|
|
||||||
if head_frames is None:
|
|
||||||
logger.error('Impossible to retrieve I-frame')
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
# Get the nearest I-frame whose timestamp ...
|
|
||||||
# TODO: wrong here ...
|
|
||||||
tail_frames = get_nearest_iframe(paths['ffprobe'], mkv, ts2, before=True)
|
|
||||||
if tail_frames is None:
|
|
||||||
logger.error('Impossible to retrieve I-frame')
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
nb_head_frames, head_iframe = head_frames
|
|
||||||
nb_tail_frames, tail_iframe = tail_frames
|
|
||||||
|
|
||||||
logger.info("Found %d frames between beginning of current part and first I-frame",
|
|
||||||
nb_head_frames)
|
|
||||||
logger.info("Found %d frames between last I-frame and end of current part",
|
|
||||||
nb_tail_frames)
|
|
||||||
|
|
||||||
head_iframe_ts = get_ts_frame(head_iframe)
|
|
||||||
if head_iframe_ts is None:
|
|
||||||
exit(-1)
|
|
||||||
tail_iframe_ts = get_ts_frame(tail_iframe)
|
|
||||||
if tail_iframe_ts is None:
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
checks.append(pos+head_iframe_ts-ts1)
|
|
||||||
|
|
||||||
subparts = []
|
|
||||||
|
|
||||||
# TODO: separate pipeline processing between coarse and not fine grain options.
|
|
||||||
|
|
||||||
# if args.coarse:
|
|
||||||
# do_coarse_processing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], input_file=mkv,
|
|
||||||
# begin=ts1, end=head_iframe_ts, nb_frames=nb_head_frames-1,
|
|
||||||
# frameRate=frameRate, files_prefix='part-%d-head' % (partnum),
|
|
||||||
# streams=streams, width=width, height=height,
|
|
||||||
# temporaries=temporaries, dump_mem_fd=args.dump)
|
|
||||||
# else:
|
|
||||||
# doFineGrainProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'],
|
|
||||||
# input_file=mkv, begin=ts1, end=head_iframe_ts,
|
|
||||||
# nb_frames=nb_head_frames-1, frameRate=frameRate,
|
|
||||||
# files_prefix='part-%d-head' % (partnum), streams=streams,
|
|
||||||
# width=width, height=height, temporaries=temporaries,
|
|
||||||
# dump_mem_fd=args.dump)
|
|
||||||
|
|
||||||
if (not args.coarse) and (nb_head_frames > args.threshold):
|
|
||||||
# We extract all frames between the beginning upto the frame that immediately preceeds
|
|
||||||
# the I-frame.
|
|
||||||
h264_head, h264_head_ts, mkv_head = extract_all_streams(ffmpeg_path=paths['ffmpeg'],
|
|
||||||
ffprobe_path=paths['ffprobe'],
|
|
||||||
input_file=mkv, begin=ts1,
|
|
||||||
end=head_iframe_ts,
|
|
||||||
nb_frames=nb_head_frames-1,
|
|
||||||
framerate=framerate,
|
|
||||||
files_prefix=f'part-{partnum:d}-head',
|
|
||||||
streams=streams, width=width,
|
|
||||||
height=height,
|
|
||||||
temporaries=temporaries,
|
|
||||||
dump_mem_fd=args.dump)
|
|
||||||
|
|
||||||
# If we are not at an exact boundary:
|
|
||||||
if mkv_head is not None:
|
|
||||||
subparts.append(mkv_head)
|
|
||||||
if h264_head is not None:
|
|
||||||
avcconfig = get_avc_config_from_h264(h264_head)
|
|
||||||
other_avc_configs.append(avcconfig)
|
|
||||||
h264parts.append(h264_head)
|
|
||||||
if h264_head_ts is not None:
|
|
||||||
h264_ts.append(h264_head_ts)
|
|
||||||
|
|
||||||
# Creating MKV file that corresponds to current part between I-frames
|
|
||||||
# Internal video with all streams (video, audio and subtitles)
|
|
||||||
internal_mkv_name = f'part-{partnum:d}-internal.mkv'
|
|
||||||
# Internal video stream as a raw H264 stream
|
|
||||||
internal_h264_name = f'part-{partnum:d}-internal.h264'
|
|
||||||
# Internal video timestamps
|
|
||||||
internal_h264_ts_name = f'part-{partnum:d}-internal-ts.txt'
|
|
||||||
# Internal video with only audio and subtitles streams
|
|
||||||
internal_novideo_mkv_name = f'part-{partnum:d}-internal-novideo.mkv'
|
|
||||||
|
|
||||||
try:
|
|
||||||
internal_mkv = open(internal_mkv_name, 'wb+')
|
|
||||||
except OSError:
|
|
||||||
logger.error('Impossible to create file: %s', internal_mkv_name)
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
internal_novideo_mkv = open(internal_novideo_mkv_name, 'wb+')
|
|
||||||
except OSError:
|
|
||||||
logger.error('Impossible to create file: %s', internal_novideo_mkv_name)
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
internal_h264 = open(internal_h264_name, 'wb+')
|
|
||||||
except OSError:
|
|
||||||
logger.error('Impossible to create file: %s', internal_h264_name)
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
internal_h264_ts = open(internal_h264_ts_name, 'w+', encoding='utf8')
|
|
||||||
except OSError:
|
|
||||||
logger.error('Impossible to create file: %s', internal_h264_ts_name)
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
# logger.info('Merge header, middle and trailer subpart into: %s' % internal_mkv_name)
|
|
||||||
# Extract internal part of MKV
|
|
||||||
extract_mkv_part(mkvmerge_path=paths['mkvmerge'], input_file=mkv, output_file=internal_mkv,
|
|
||||||
begin=head_iframe_ts, end=tail_iframe_ts)
|
|
||||||
|
|
||||||
# Extract video stream of internal part as a raw H264 and its timestamps.
|
|
||||||
logger.info('Extract video track as raw H264 file.')
|
|
||||||
extract_track_from_mkv(mkvextract_path=paths['mkvextract'], input_file=internal_mkv,
|
|
||||||
index=0, output_file=internal_h264, timestamps=internal_h264_ts)
|
|
||||||
|
|
||||||
# Remove video track from internal part of MKV
|
|
||||||
logger.info('Remove video track from %s', internal_mkv_name)
|
|
||||||
remove_video_tracks_from_mkv(mkvmerge_path=paths['mkvmerge'], input_file=internal_mkv,
|
|
||||||
output_file=internal_novideo_mkv)
|
|
||||||
|
|
||||||
temporaries.append(internal_mkv)
|
|
||||||
temporaries.append(internal_h264)
|
|
||||||
temporaries.append(internal_h264_ts)
|
|
||||||
temporaries.append(internal_novideo_mkv)
|
|
||||||
|
|
||||||
h264parts.append(internal_h264)
|
|
||||||
h264_ts.append(internal_h264_ts)
|
|
||||||
subparts.append(internal_novideo_mkv)
|
|
||||||
|
|
||||||
if (not args.coarse) and (nb_tail_frames > args.threshold):
|
|
||||||
# We extract all frames between the I-frame (including it) upto the end.
|
|
||||||
h264_tail, h264_tail_ts, mkv_tail = extract_all_streams(ffmpeg_path=paths['ffmpeg'],
|
|
||||||
ffprobe_path=paths['ffprobe'],
|
|
||||||
input_file=mkv, begin=tail_iframe_ts,
|
|
||||||
end=ts2, nb_frames=nb_tail_frames,
|
|
||||||
framerate=framerate,
|
|
||||||
files_prefix=f'part-{partnum:d}-tail',
|
|
||||||
streams=streams,
|
|
||||||
width=width, height=height,
|
|
||||||
temporaries=temporaries,
|
|
||||||
dump_mem_fd=args.dump)
|
|
||||||
|
|
||||||
if mkv_tail is not None:
|
|
||||||
subparts.append(mkv_tail)
|
|
||||||
if h264_tail is not None:
|
|
||||||
avcconfig = get_avc_config_from_h264(h264_tail)
|
|
||||||
other_avc_configs.append(avcconfig)
|
|
||||||
h264parts.append(h264_tail)
|
|
||||||
if h264_tail_ts is not None:
|
|
||||||
h264_ts.append(h264_tail_ts)
|
|
||||||
|
|
||||||
logger.info('Merging MKV: %s', subparts)
|
|
||||||
|
|
||||||
part = merge_mkvs(mkvmerge_path=paths['mkvmerge'], inputs=subparts,
|
|
||||||
output_name=f'part-{partnum:d}.mkv', concatenate=True)
|
|
||||||
mkvparts.append(part)
|
|
||||||
temporaries.append(part)
|
|
||||||
|
|
||||||
pos = pos+tail_iframe_ts-ts1
|
|
||||||
|
|
||||||
# We need to check the end also
|
|
||||||
checks.append(pos)
|
|
||||||
|
|
||||||
# When using coarse option there is a single AVC configuration.
|
|
||||||
for avc_config in other_avc_configs:
|
|
||||||
main_avc_config.merge(avc_config)
|
|
||||||
logger.debug('Merged AVC configuration: %s', main_avc_config)
|
|
||||||
|
|
||||||
nb_mkv_parts = len(mkvparts)
|
|
||||||
if nb_mkv_parts > 0:
|
|
||||||
try:
|
|
||||||
full_h264 = open(f'{basename}-full.h264', 'wb+')
|
|
||||||
except OSError:
|
|
||||||
logger.error('Impossible to create file full H264 stream.')
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
logger.info('Merging all H264 tracks')
|
|
||||||
concatenate_h264_parts(h264parts=h264parts, output=full_h264)
|
|
||||||
temporaries.append(full_h264)
|
|
||||||
|
|
||||||
try:
|
|
||||||
full_h264_ts = open(f'{basename}-ts.txt', 'w+', encoding='utf8')
|
|
||||||
except OSError:
|
|
||||||
logger.error('Impossible to create file containing all video timestamps.')
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
logger.info('Merging H264 timestamps')
|
|
||||||
concatenate_h264_ts_parts(h264_ts_parts=h264_ts, output=full_h264_ts)
|
|
||||||
temporaries.append(full_h264_ts)
|
|
||||||
|
|
||||||
final_novideo_name = f'{basename}-novideo.mkv'
|
|
||||||
final_with_video_name = f'{basename}-video.mkv'
|
|
||||||
|
|
||||||
if nb_mkv_parts > 1:
|
|
||||||
logger.info('Merging all audio and subtitles parts: %s', mkvparts)
|
|
||||||
merge_mkvs(mkvmerge_path=paths['mkvmerge'], inputs=mkvparts, output_name=final_novideo_name,
|
|
||||||
concatenate=True)
|
|
||||||
elif nb_mkv_parts == 1:
|
|
||||||
copyfile('part-1.mkv', final_novideo_name)
|
|
||||||
else:
|
|
||||||
logger.info("Nothing else to do.")
|
|
||||||
copyfile(mkvfilename, final_with_video_name)
|
|
||||||
|
|
||||||
if nb_mkv_parts >=1 :
|
|
||||||
try:
|
|
||||||
final_novideo = open(final_novideo_name, 'rb')
|
|
||||||
except OSError:
|
|
||||||
logger.error('Impossible to open file: %s.', final_novideo_name)
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
temporaries.append(final_novideo)
|
|
||||||
|
|
||||||
full_h264_ts.seek(0)
|
|
||||||
|
|
||||||
logger.info('Merging final video track and all other tracks together')
|
|
||||||
final_with_video = merge_mkvs(mkvmerge_path=paths['mkvmerge'], inputs=[full_h264,
|
|
||||||
final_novideo],
|
|
||||||
output_name=final_with_video_name, concatenate=False,
|
|
||||||
timestamps={0: full_h264_ts})
|
|
||||||
final_codec_private_data = dump_codec_private_data(main_avc_config)
|
|
||||||
logger.debug('Final codec private data: %s', hexdump.dump(final_codec_private_data,
|
|
||||||
sep=':'))
|
|
||||||
logger.info('Changing codec private data with the new one.')
|
|
||||||
change_codec_private_data(paths['mkvinfo'], final_with_video, final_codec_private_data)
|
|
||||||
|
|
||||||
if args.srt:
|
|
||||||
if not all_optional_tools:
|
|
||||||
logger.warning("Missing tools for extracting subtitles.")
|
|
||||||
move(final_with_video_name, args.output_file)
|
|
||||||
else:
|
|
||||||
# Final cut is not any more the final step.
|
|
||||||
temporaries.append(final_with_video)
|
|
||||||
duration = get_movie_duration(paths['ffprobe'], final_with_video)
|
|
||||||
supported_langs = get_tesseract_supported_lang(paths['tesseract'])
|
|
||||||
logger.info('Supported lang: %s', supported_langs)
|
|
||||||
logger.info('Find subtitles tracks and language.')
|
|
||||||
subtitles = find_subtitles_tracks(paths['ffprobe'], final_with_video)
|
|
||||||
logger.info(subtitles)
|
|
||||||
sts = {}
|
|
||||||
for subtitle in subtitles:
|
|
||||||
index = subtitle['index']
|
|
||||||
if 'tags' in subtitle:
|
|
||||||
if 'language' in subtitle['tags']:
|
|
||||||
lang = subtitle['tags']['language']
|
|
||||||
if lang in sts:
|
|
||||||
sts[lang].append(index)
|
|
||||||
else:
|
|
||||||
sts[lang] = [index]
|
|
||||||
else:
|
|
||||||
logger.error("Dropping subtitle: %s because it is missing language\
|
|
||||||
indication", subtitle)
|
|
||||||
else:
|
|
||||||
logger.error("Dropping subtitle: %s because it is missing language indication",
|
|
||||||
subtitle)
|
|
||||||
|
|
||||||
logger.info(sts)
|
|
||||||
if len(sts) > 0:
|
|
||||||
logger.info('Supported languages: %s', supported_langs)
|
|
||||||
list_of_subtitles = extract_srt(paths['mkvextract'], final_with_video_name, sts,
|
|
||||||
supported_langs)
|
|
||||||
logger.info(list_of_subtitles)
|
|
||||||
for idx_name, sub_name, _, _ in list_of_subtitles:
|
|
||||||
try:
|
|
||||||
idx = open(idx_name,'rb')
|
|
||||||
except OSError:
|
|
||||||
logger.error("Impossible to open %s.", idx_name)
|
|
||||||
exit(-1)
|
|
||||||
try:
|
|
||||||
sub = open(sub_name,'rb')
|
|
||||||
except OSError:
|
|
||||||
logger.error("Impossible to open %s.", sub_name)
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
temporaries.append(idx)
|
|
||||||
temporaries.append(sub)
|
|
||||||
|
|
||||||
ocr = do_ocr(paths['vobsubocr'], list_of_subtitles, duration, temporaries,
|
|
||||||
args.dump)
|
|
||||||
logger.info(ocr)
|
|
||||||
|
|
||||||
# Remux SRT subtitles
|
|
||||||
remux_srt_subtitles(paths['mkvmerge'], final_with_video, args.output_file, ocr)
|
|
||||||
else:
|
|
||||||
copyfile(final_with_video_name, args.output_file)
|
|
||||||
else:
|
|
||||||
move(final_with_video_name, args.output_file)
|
|
||||||
|
|
||||||
if not args.keep:
|
|
||||||
logger.info("Cleaning temporary files")
|
logger.info("Cleaning temporary files")
|
||||||
for f in temporaries:
|
temporaries.cleanup()
|
||||||
path = os.path.realpath(f.name)
|
|
||||||
logger.info("Removing: %s", path)
|
|
||||||
f.close()
|
|
||||||
unlink(path)
|
|
||||||
|
|
||||||
d = datetime(1,1,1)
|
|
||||||
for c in checks:
|
|
||||||
logger.info("Please check cut smoothness at %s", (c+d).strftime("%H:%M:%S"))
|
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
#
|
||||||
|
# Copyright (C) 2026 Frédéric Tronel
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from typing import IO
|
||||||
|
|
||||||
|
from tscut.exceptions import TemporaryFileError
|
||||||
|
from tscut.tools.mkvtoolnix import extract_mkv_part
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# TODO: finish this procedure
|
||||||
|
def do_coarse_processing(ffmpeg_path:str, ffprobe_path:str, mkvmerge_path:str,
|
||||||
|
input_file: IO[bytes], begin, end, nb_frames, framerate,
|
||||||
|
files_prefix, streams, width, height, temporaries, dump_mem_fd) -> None:
|
||||||
|
# Internal video with all streams (video, audio and subtitles)
|
||||||
|
internal_mkv_name = f'{files_prefix}.mkv'
|
||||||
|
|
||||||
|
try:
|
||||||
|
internal_mkv = open(internal_mkv_name, 'wb+')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to create {internal_mkv_name}") from e
|
||||||
|
|
||||||
|
# Extract internal part of MKV
|
||||||
|
extract_mkv_part(mkvmerge_path=mkvmerge_path, input_file=input_file, output_file=internal_mkv,
|
||||||
|
begin=begin, end=end)
|
||||||
|
|
||||||
|
temporaries.add(internal_mkv)
|
||||||
@@ -0,0 +1,365 @@
|
|||||||
|
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
#
|
||||||
|
# Copyright (C) 2026 Frédéric Tronel
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import timedelta
|
||||||
|
from os import (
|
||||||
|
SEEK_SET,
|
||||||
|
fstat,
|
||||||
|
lseek,
|
||||||
|
read,
|
||||||
|
write,
|
||||||
|
)
|
||||||
|
from shutil import copyfile
|
||||||
|
from typing import BinaryIO, Sequence, TextIO
|
||||||
|
|
||||||
|
import hexdump
|
||||||
|
from tqdm import tqdm
|
||||||
|
from typeguard import typechecked
|
||||||
|
|
||||||
|
from tscut.exceptions import InvalidMediaError, TemporaryFileError
|
||||||
|
from tscut.h264.avc import get_avc_config_from_h264
|
||||||
|
from tscut.matroska.codec import dump_codec_private_data
|
||||||
|
from tscut.models import CutResult, PreparedMedia, ProcessingOptions
|
||||||
|
from tscut.temporaries import TemporaryFiles
|
||||||
|
from tscut.tools.ffmpeg import extract_all_streams
|
||||||
|
from tscut.tools.ffprobe import (
|
||||||
|
get_nearest_iframe,
|
||||||
|
)
|
||||||
|
from tscut.tools.mkvtoolnix import (
|
||||||
|
change_codec_private_data,
|
||||||
|
extract_mkv_part,
|
||||||
|
extract_track_from_mkv,
|
||||||
|
merge_mkvs,
|
||||||
|
remove_video_tracks_from_mkv,
|
||||||
|
)
|
||||||
|
from tscut.tools.timeframe import get_ts_frame
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@typechecked
|
||||||
|
def concatenate_h264_parts(h264parts: Sequence[BinaryIO], output: BinaryIO) -> None:
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
total_length = 0
|
||||||
|
for h264 in h264parts:
|
||||||
|
fd = h264.fileno()
|
||||||
|
total_length += fstat(fd).st_size
|
||||||
|
|
||||||
|
logger.info('Total length: %d', total_length)
|
||||||
|
|
||||||
|
outfd = output.fileno()
|
||||||
|
lseek(outfd, 0, SEEK_SET)
|
||||||
|
|
||||||
|
pb = tqdm(total=total_length, unit='bytes', desc='Concatenation')
|
||||||
|
for h264 in h264parts:
|
||||||
|
fd = h264.fileno()
|
||||||
|
lseek(fd, 0, SEEK_SET)
|
||||||
|
while True:
|
||||||
|
buf = read(fd, 1000000)
|
||||||
|
if buf is None or len(buf) == 0:
|
||||||
|
break
|
||||||
|
pos = 0
|
||||||
|
while pos < len(buf):
|
||||||
|
nb_bytes = write(outfd, buf[pos:])
|
||||||
|
pb.update(nb_bytes)
|
||||||
|
pos += nb_bytes
|
||||||
|
|
||||||
|
def concatenate_h264_ts_parts(h264_ts_parts: Sequence[TextIO], output: TextIO) -> None:
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
header = '# timestamp format v2\n'
|
||||||
|
output.write(header)
|
||||||
|
|
||||||
|
last = 0.
|
||||||
|
first = True
|
||||||
|
for part in h264_ts_parts:
|
||||||
|
if first:
|
||||||
|
offset = last
|
||||||
|
else:
|
||||||
|
# TODO: take framerate into account
|
||||||
|
offset = last + 40
|
||||||
|
logger.debug('Parsing file: %s. Offset=%d', part, offset)
|
||||||
|
isheader = part.readline()
|
||||||
|
if (not isheader) or (isheader != header):
|
||||||
|
raise InvalidMediaError(f"Impossible to find a valid header: {isheader}")
|
||||||
|
while True:
|
||||||
|
line = part.readline()
|
||||||
|
if not line:
|
||||||
|
break
|
||||||
|
ts = offset + float(line)
|
||||||
|
last = max(last,ts)
|
||||||
|
output.write(f'{ts:f}\n')
|
||||||
|
if first:
|
||||||
|
first = False
|
||||||
|
|
||||||
|
def cut_recording(media: PreparedMedia, options: ProcessingOptions,
|
||||||
|
temporaries: TemporaryFiles) -> CutResult:
|
||||||
|
|
||||||
|
# Pour chaque portion
|
||||||
|
partnum = 0
|
||||||
|
mkvparts = []
|
||||||
|
h264parts = []
|
||||||
|
h264_ts = []
|
||||||
|
checks = []
|
||||||
|
pos = timedelta()
|
||||||
|
other_avc_configs = []
|
||||||
|
mkvfilename = media.basename+'.mkv'
|
||||||
|
|
||||||
|
|
||||||
|
for ts1, ts2 in options.parts:
|
||||||
|
# TODO: translate comment in english
|
||||||
|
# Trouver l'estampille de la trame 'I' la plus proche (mais postérieure) au début
|
||||||
|
# de la portion.
|
||||||
|
# Trouver l'estampille de la trame 'I' la plus proche (mais antérieure) à la fin
|
||||||
|
# de la portion.
|
||||||
|
# On a alors
|
||||||
|
# debut ----- trame --------- trame --------- fin fin+1
|
||||||
|
# 'B/P' 'B/P'* 'I' 'I' 'B/P'* 'B/P' 'I/B/P'
|
||||||
|
# Si la trame de début est déjà 'I', il n'y a rien à faire.
|
||||||
|
# Sinon on extrait les trames 'B' ou 'P' depuis le début jusqu'à la trame 'I' non incluse.
|
||||||
|
# Si la trame de fin précède une trame I, on n'a rien à faire.
|
||||||
|
# Sinon on extrait toutes les trames depuis la dernière trame I jusqu'à la trame de fin.
|
||||||
|
|
||||||
|
partnum = partnum + 1
|
||||||
|
|
||||||
|
# Get the nearest I-frame whose timestamp is greater or equal to the beginning.
|
||||||
|
head_frames = get_nearest_iframe(options.tools_paths['ffprobe'], media.movie, ts1,
|
||||||
|
before=False)
|
||||||
|
if head_frames is None:
|
||||||
|
raise InvalidMediaError("Impossible to retrieve first I-frame")
|
||||||
|
|
||||||
|
# Get the nearest I-frame whose timestamp ...
|
||||||
|
# TODO: wrong here ...
|
||||||
|
tail_frames = get_nearest_iframe(options.tools_paths['ffprobe'], media.movie,
|
||||||
|
ts2, before=True)
|
||||||
|
if tail_frames is None:
|
||||||
|
raise InvalidMediaError("Impossible to retrieve last I-frame")
|
||||||
|
|
||||||
|
nb_head_frames, head_iframe = head_frames
|
||||||
|
nb_tail_frames, tail_iframe = tail_frames
|
||||||
|
|
||||||
|
logger.info("Found %d frames between beginning of current part and first I-frame",
|
||||||
|
nb_head_frames)
|
||||||
|
logger.info("Found %d frames between last I-frame and end of current part",
|
||||||
|
nb_tail_frames)
|
||||||
|
try:
|
||||||
|
head_iframe_ts = get_ts_frame(head_iframe)
|
||||||
|
except InvalidMediaError:
|
||||||
|
raise InvalidMediaError("Impossible to retrieve timestamp of first i-frame.")
|
||||||
|
try:
|
||||||
|
tail_iframe_ts = get_ts_frame(tail_iframe)
|
||||||
|
except InvalidMediaError:
|
||||||
|
raise InvalidMediaError("Impossible to retrieve timestamp of final i-frame.")
|
||||||
|
|
||||||
|
assert head_iframe_ts <= tail_iframe_ts
|
||||||
|
|
||||||
|
checks.append(pos+head_iframe_ts-ts1)
|
||||||
|
|
||||||
|
subparts = []
|
||||||
|
|
||||||
|
# TODO: separate pipeline processing between coarse and not fine grain options.
|
||||||
|
|
||||||
|
# if args.coarse:
|
||||||
|
# do_coarse_processing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'], input_file=mkv,
|
||||||
|
# begin=ts1, end=head_iframe_ts, nb_frames=nb_head_frames-1,
|
||||||
|
# frameRate=frameRate, files_prefix='part-%d-head' % (partnum),
|
||||||
|
# streams=streams, width=width, height=height,
|
||||||
|
# temporaries=temporaries, dump_mem_fd=args.dump)
|
||||||
|
# else:
|
||||||
|
# doFineGrainProcessing(ffmpeg=paths['ffmpeg'], ffprobe=paths['ffprobe'],
|
||||||
|
# input_file=mkv, begin=ts1, end=head_iframe_ts,
|
||||||
|
# nb_frames=nb_head_frames-1, frameRate=frameRate,
|
||||||
|
# files_prefix='part-%d-head' % (partnum), streams=streams,
|
||||||
|
# width=width, height=height, temporaries=temporaries,
|
||||||
|
# dump_mem_fd=args.dump)
|
||||||
|
|
||||||
|
if (not options.coarse) and (nb_head_frames > options.threshold):
|
||||||
|
# We extract all frames between the beginning upto the frame that immediately preceeds
|
||||||
|
# the I-frame.
|
||||||
|
h264_head, h264_head_ts, mkv_head = extract_all_streams(
|
||||||
|
ffmpeg_path=options.tools_paths['ffmpeg'],
|
||||||
|
ffprobe_path=options.tools_paths['ffprobe'],
|
||||||
|
input_file=media.movie, begin=ts1,
|
||||||
|
end=head_iframe_ts,
|
||||||
|
nb_frames=nb_head_frames-1,
|
||||||
|
framerate=media.framerate,
|
||||||
|
files_prefix=f'part-{partnum:d}-head',
|
||||||
|
streams=media.streams, width=media.width,
|
||||||
|
height=media.height,
|
||||||
|
temporaries=temporaries,
|
||||||
|
dump_mem_fd=options.dump_memory)
|
||||||
|
|
||||||
|
# If we are not at an exact boundary:
|
||||||
|
if mkv_head is not None:
|
||||||
|
subparts.append(mkv_head)
|
||||||
|
if h264_head is not None:
|
||||||
|
avcconfig = get_avc_config_from_h264(h264_head)
|
||||||
|
other_avc_configs.append(avcconfig)
|
||||||
|
h264parts.append(h264_head)
|
||||||
|
if h264_head_ts is not None:
|
||||||
|
h264_ts.append(h264_head_ts)
|
||||||
|
|
||||||
|
if head_iframe_ts < tail_iframe_ts:
|
||||||
|
|
||||||
|
# Creating MKV file that corresponds to current part between I-frames
|
||||||
|
# Internal video with all streams (video, audio and subtitles)
|
||||||
|
internal_mkv_name = f'part-{partnum:d}-internal.mkv'
|
||||||
|
# Internal video stream as a raw H264 stream
|
||||||
|
internal_h264_name = f'part-{partnum:d}-internal.h264'
|
||||||
|
# Internal video timestamps
|
||||||
|
internal_h264_ts_name = f'part-{partnum:d}-internal-ts.txt'
|
||||||
|
# Internal video with only audio and subtitles streams
|
||||||
|
internal_novideo_mkv_name = f'part-{partnum:d}-internal-novideo.mkv'
|
||||||
|
|
||||||
|
try:
|
||||||
|
internal_mkv = open(internal_mkv_name, 'wb+')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to create file: {internal_mkv_name}") from e
|
||||||
|
|
||||||
|
try:
|
||||||
|
internal_novideo_mkv = open(internal_novideo_mkv_name, 'wb+')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to create file: \
|
||||||
|
{internal_novideo_mkv_name}") from e
|
||||||
|
|
||||||
|
try:
|
||||||
|
internal_h264 = open(internal_h264_name, 'wb+')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to create file: {internal_h264_name}") from e
|
||||||
|
|
||||||
|
try:
|
||||||
|
internal_h264_ts = open(internal_h264_ts_name, 'w+', encoding='utf8')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to create file: {internal_h264_ts_name}") from e
|
||||||
|
|
||||||
|
# logger.info('Merge header, middle and trailer subpart into: %s' % internal_mkv_name)
|
||||||
|
# Extract internal part of MKV
|
||||||
|
extract_mkv_part(mkvmerge_path=options.tools_paths['mkvmerge'], input_file=media.movie,
|
||||||
|
output_file=internal_mkv, begin=head_iframe_ts, end=tail_iframe_ts)
|
||||||
|
|
||||||
|
# Extract video stream of internal part as a raw H264 and its timestamps.
|
||||||
|
logger.info('Extract video track as raw H264 file.')
|
||||||
|
extract_track_from_mkv(mkvextract_path=options.tools_paths['mkvextract'],
|
||||||
|
input_file=internal_mkv, index=0, output_file=internal_h264,
|
||||||
|
timestamps=internal_h264_ts)
|
||||||
|
|
||||||
|
# Remove video track from internal part of MKV
|
||||||
|
logger.info('Remove video track from %s', internal_mkv_name)
|
||||||
|
remove_video_tracks_from_mkv(mkvmerge_path=options.tools_paths['mkvmerge'],
|
||||||
|
input_file=internal_mkv, output_file=internal_novideo_mkv)
|
||||||
|
|
||||||
|
temporaries.add(internal_mkv)
|
||||||
|
temporaries.add(internal_h264)
|
||||||
|
temporaries.add(internal_h264_ts)
|
||||||
|
temporaries.add(internal_novideo_mkv)
|
||||||
|
|
||||||
|
h264parts.append(internal_h264)
|
||||||
|
h264_ts.append(internal_h264_ts)
|
||||||
|
subparts.append(internal_novideo_mkv)
|
||||||
|
|
||||||
|
if (not options.coarse) and (nb_tail_frames > options.threshold):
|
||||||
|
# We extract all frames between the I-frame (including it) upto the end.
|
||||||
|
h264_tail, h264_tail_ts, mkv_tail = extract_all_streams(
|
||||||
|
ffmpeg_path=options.tools_paths['ffmpeg'],
|
||||||
|
ffprobe_path=options.tools_paths['ffprobe'],
|
||||||
|
input_file=media.movie, begin=tail_iframe_ts,
|
||||||
|
end=ts2, nb_frames=nb_tail_frames,
|
||||||
|
framerate=media.framerate,
|
||||||
|
files_prefix=f'part-{partnum:d}-tail',
|
||||||
|
streams=media.streams,
|
||||||
|
width=media.width, height=media.height,
|
||||||
|
temporaries=temporaries,
|
||||||
|
dump_mem_fd=options.dump_memory)
|
||||||
|
|
||||||
|
if mkv_tail is not None:
|
||||||
|
subparts.append(mkv_tail)
|
||||||
|
if h264_tail is not None:
|
||||||
|
avcconfig = get_avc_config_from_h264(h264_tail)
|
||||||
|
other_avc_configs.append(avcconfig)
|
||||||
|
h264parts.append(h264_tail)
|
||||||
|
if h264_tail_ts is not None:
|
||||||
|
h264_ts.append(h264_tail_ts)
|
||||||
|
|
||||||
|
logger.info('Merging MKV: %s', subparts)
|
||||||
|
|
||||||
|
part = merge_mkvs(mkvmerge_path=options.tools_paths['mkvmerge'], inputs=subparts,
|
||||||
|
output_name=f'part-{partnum:d}.mkv', concatenate=True)
|
||||||
|
mkvparts.append(part)
|
||||||
|
temporaries.add(part)
|
||||||
|
|
||||||
|
pos = pos+tail_iframe_ts-ts1
|
||||||
|
|
||||||
|
# We need to check the end also
|
||||||
|
checks.append(pos)
|
||||||
|
|
||||||
|
# When using coarse option there is a single AVC configuration.
|
||||||
|
for avc_config in other_avc_configs:
|
||||||
|
media.avc_config.merge(avc_config)
|
||||||
|
logger.debug('Merged AVC configuration: %s', media.avc_config)
|
||||||
|
|
||||||
|
nb_mkv_parts = len(mkvparts)
|
||||||
|
if nb_mkv_parts > 0:
|
||||||
|
try:
|
||||||
|
full_h264 = open(f'{media.basename}-full.h264', 'wb+')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError("Impossible to create file full H264 stream.") from e
|
||||||
|
|
||||||
|
logger.info('Merging all H264 tracks')
|
||||||
|
concatenate_h264_parts(h264parts=h264parts, output=full_h264)
|
||||||
|
temporaries.add(full_h264)
|
||||||
|
|
||||||
|
try:
|
||||||
|
full_h264_ts = open(f'{media.basename}-ts.txt', 'w+', encoding='utf8')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError("Impossible to create file containing all video timestamps.")\
|
||||||
|
from e
|
||||||
|
|
||||||
|
logger.info('Merging H264 timestamps')
|
||||||
|
concatenate_h264_ts_parts(h264_ts_parts=h264_ts, output=full_h264_ts)
|
||||||
|
temporaries.add(full_h264_ts)
|
||||||
|
|
||||||
|
final_novideo_name = f'{media.basename}-novideo.mkv'
|
||||||
|
final_with_video_name = f'{media.basename}-video.mkv'
|
||||||
|
|
||||||
|
if nb_mkv_parts > 1:
|
||||||
|
logger.info('Merging all audio and subtitles parts: %s', mkvparts)
|
||||||
|
merge_mkvs(mkvmerge_path=options.tools_paths['mkvmerge'], inputs=mkvparts,
|
||||||
|
output_name=final_novideo_name, concatenate=True)
|
||||||
|
elif nb_mkv_parts == 1:
|
||||||
|
copyfile('part-1.mkv', final_novideo_name)
|
||||||
|
else:
|
||||||
|
logger.info("Nothing else to do.")
|
||||||
|
copyfile(mkvfilename, final_with_video_name)
|
||||||
|
|
||||||
|
if nb_mkv_parts >=1 :
|
||||||
|
try:
|
||||||
|
final_novideo = open(final_novideo_name, 'rb')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to open file: {final_novideo_name}") from e
|
||||||
|
|
||||||
|
temporaries.add(final_novideo)
|
||||||
|
|
||||||
|
full_h264_ts.seek(0)
|
||||||
|
|
||||||
|
logger.info('Merging final video track and all other tracks together')
|
||||||
|
final_with_video = merge_mkvs(mkvmerge_path=options.tools_paths['mkvmerge'],
|
||||||
|
inputs=[full_h264, final_novideo],
|
||||||
|
output_name=final_with_video_name, concatenate=False,
|
||||||
|
timestamps={0: full_h264_ts})
|
||||||
|
final_codec_private_data = dump_codec_private_data(media.avc_config)
|
||||||
|
logger.debug('Final codec private data: %s', hexdump.dump(final_codec_private_data,
|
||||||
|
sep=':'))
|
||||||
|
logger.info('Changing codec private data with the new one.')
|
||||||
|
change_codec_private_data(options.tools_paths['mkvinfo'], final_with_video,
|
||||||
|
final_codec_private_data)
|
||||||
|
|
||||||
|
return CutResult(
|
||||||
|
filename = final_with_video_name,
|
||||||
|
movie = final_with_video,
|
||||||
|
check_positions = checks
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
|
||||||
|
Useful SPS/PPS discussion:
|
||||||
|
1.https://copyprogramming.com/howto/including-sps-and-pps-in-a-raw-h264-track
|
||||||
|
2. https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390
|
||||||
|
|
||||||
|
Strategy: a possible way of handling multiple SPS/PPS gracefully.
|
||||||
|
Encode each head and trailer with FFMPEG using only I-frame (to be sure the NAL unit will never refer to another image).
|
||||||
|
Encode using an different SPS-ID all of them (using sps-id parameter of libx264 library, e.g 1 instead of 0).
|
||||||
|
For the video track produce only a raw H264 file and a file containing timestamps of the different frames.
|
||||||
|
For the rest of the tracks (audio, subtitles) produce directly a MKV (this is already done).
|
||||||
|
Concatenate all raw H264 in a giant one (like cat), and the same for timestamps of video frames (to keep sound and video synchronized).
|
||||||
|
Then use mkvmerge to remux the H264 track and the rest of tracks.
|
||||||
|
MKVmerge "concatenate" subcommand is able to concatenate different SPS/PPS data into a bigger Private Codec Data.
|
||||||
|
However, this is proved to be not reliable.
|
||||||
|
Sometimes it results in a AVC context containing a single SPS/PPS.
|
||||||
|
So we have to rely on a manual parsing of the H264 AVC context of original movie and the ones produced for headers and trailers, and then merging them into a bigger AVC context.
|
||||||
|
Then finally, change the Private Codec Data in the final MKV.
|
||||||
|
|
||||||
|
|
||||||
|
Extract SPS/PPS:
|
||||||
|
1. https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390
|
||||||
|
```bash
|
||||||
|
ffmpeg -i <InputFile (before concatenation)> -c:v copy -an -sn -bsf:v trace_headers -t 0.01 -report -loglevel 0 -f null -
|
||||||
|
``
|
||||||
|
|
||||||
|
|
||||||
@@ -3,10 +3,6 @@
|
|||||||
# Copyright (C) 2026 Frédéric Tronel
|
# Copyright (C) 2026 Frédéric Tronel
|
||||||
|
|
||||||
|
|
||||||
from typing import IO
|
|
||||||
|
|
||||||
from typeguard import typechecked
|
|
||||||
|
|
||||||
class TSCutError(Exception):
|
class TSCutError(Exception):
|
||||||
"""Base exception for tscut."""
|
"""Base exception for tscut."""
|
||||||
|
|
||||||
@@ -18,6 +14,11 @@ class MissingToolError(TSCutError):
|
|||||||
class ExternalToolError(TSCutError):
|
class ExternalToolError(TSCutError):
|
||||||
"""Raised when an external tool fails."""
|
"""Raised when an external tool fails."""
|
||||||
|
|
||||||
|
|
||||||
class InvalidMediaError(TSCutError):
|
class InvalidMediaError(TSCutError):
|
||||||
"""Raised when input media cannot be processed."""
|
"""Raised when input media cannot be processed."""
|
||||||
|
|
||||||
|
class TemporaryFileError(TSCutError):
|
||||||
|
"""Raised when the manipulation of temporary file goes wrong"""
|
||||||
|
|
||||||
|
class UnimplementedFeatureError(TSCutError):
|
||||||
|
"""Raised when a rare feature is encountered and not yet implemented"""
|
||||||
|
|||||||
@@ -8,26 +8,24 @@ from dataclasses import dataclass, field
|
|||||||
from math import floor
|
from math import floor
|
||||||
from typing import IO
|
from typing import IO
|
||||||
|
|
||||||
|
|
||||||
from typeguard import typechecked
|
|
||||||
import hexdump
|
import hexdump
|
||||||
|
from typeguard import typechecked
|
||||||
|
|
||||||
from tscut.h264.bitstream import (
|
from tscut.h264.bitstream import (
|
||||||
|
rbsp_to_sodb,
|
||||||
read_bit,
|
read_bit,
|
||||||
read_bits,
|
read_bits,
|
||||||
read_byte,
|
read_byte,
|
||||||
read_long,
|
read_long,
|
||||||
read_word,
|
read_word,
|
||||||
|
sodb_to_rbsp,
|
||||||
write_bits,
|
write_bits,
|
||||||
write_byte,
|
write_byte,
|
||||||
write_word,
|
write_word,
|
||||||
sodb_to_rbsp,
|
|
||||||
rbsp_to_sodb
|
|
||||||
)
|
)
|
||||||
|
|
||||||
from tscut.h264.parameters import (
|
from tscut.h264.parameters import (
|
||||||
SPS,
|
|
||||||
PPS,
|
PPS,
|
||||||
|
SPS,
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -243,7 +241,7 @@ class AVCDecoderConfiguration:
|
|||||||
# TODO: do the same with extended SPS !
|
# TODO: do the same with extended SPS !
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def parse_codec_private(codec_private_data: bytes) -> AVCDecoderConfiguration:
|
def parse_codec_private(codec_private_data: bytes | bytearray) -> AVCDecoderConfiguration:
|
||||||
if codec_private_data[0] != 0x63:
|
if codec_private_data[0] != 0x63:
|
||||||
raise ValueError(f'Matroska header is wrong: {codec_private_data[0]:x}')
|
raise ValueError(f'Matroska header is wrong: {codec_private_data[0]:x}')
|
||||||
if codec_private_data[1] != 0xA2:
|
if codec_private_data[1] != 0xA2:
|
||||||
|
|||||||
+14
-13
@@ -10,14 +10,15 @@
|
|||||||
import logging
|
import logging
|
||||||
from math import floor, log
|
from math import floor, log
|
||||||
|
|
||||||
from typeguard import typechecked
|
|
||||||
import hexdump
|
import hexdump
|
||||||
|
from typeguard import typechecked
|
||||||
|
|
||||||
|
from tscut.exceptions import UnimplementedFeatureError
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def read_bit(buf:bytes, bit_position: int) -> tuple[int, int]:
|
def read_bit(buf:bytes|bytearray, bit_position: int) -> tuple[int, int]:
|
||||||
"""
|
"""
|
||||||
Read a single bit from a byte buffer.
|
Read a single bit from a byte buffer.
|
||||||
|
|
||||||
@@ -40,7 +41,7 @@ def read_bit(buf:bytes, bit_position: int) -> tuple[int, int]:
|
|||||||
return bit_position+1, bit
|
return bit_position+1, bit
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def read_boolean(buf:bytes, bit_position: int) -> tuple[int, bool]:
|
def read_boolean(buf:bytes|bytearray, bit_position: int) -> tuple[int, bool]:
|
||||||
"""
|
"""
|
||||||
Read a boolean value from a byte buffer.
|
Read a boolean value from a byte buffer.
|
||||||
|
|
||||||
@@ -60,7 +61,7 @@ def read_boolean(buf:bytes, bit_position: int) -> tuple[int, bool]:
|
|||||||
return bit_position, b==1
|
return bit_position, b==1
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def read_bits(buf:bytes, bit_position: int, nb_bits: int) -> tuple[int, int]:
|
def read_bits(buf:bytes|bytearray, bit_position: int, nb_bits: int) -> tuple[int, int]:
|
||||||
v = 0
|
v = 0
|
||||||
for _ in range(nb_bits):
|
for _ in range(nb_bits):
|
||||||
bit_position, bit = read_bit(buf, bit_position)
|
bit_position, bit = read_bit(buf, bit_position)
|
||||||
@@ -68,22 +69,22 @@ def read_bits(buf:bytes, bit_position: int, nb_bits: int) -> tuple[int, int]:
|
|||||||
return bit_position, v
|
return bit_position, v
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def read_byte(buf:bytes, bit_position: int) -> tuple[int, int]:
|
def read_byte(buf:bytes|bytearray, bit_position: int) -> tuple[int, int]:
|
||||||
bit_position, b = read_bits(buf, bit_position, 8)
|
bit_position, b = read_bits(buf, bit_position, 8)
|
||||||
return bit_position, b
|
return bit_position, b
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def read_word(buf:bytes, bit_position: int) -> tuple[int, int]:
|
def read_word(buf:bytes|bytearray, bit_position: int) -> tuple[int, int]:
|
||||||
bit_position, w = read_bits(buf, bit_position, 16)
|
bit_position, w = read_bits(buf, bit_position, 16)
|
||||||
return bit_position, w
|
return bit_position, w
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def read_long(buf:bytes, bit_position: int) -> tuple[int, int]:
|
def read_long(buf:bytes|bytearray, bit_position: int) -> tuple[int, int]:
|
||||||
bit_position, value = read_bits(buf, bit_position, 32)
|
bit_position, value = read_bits(buf, bit_position, 32)
|
||||||
return bit_position, value
|
return bit_position, value
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def read_unsigned_exp_golomb(buf:bytes, bit_position: int) -> tuple[int, int]:
|
def read_unsigned_exp_golomb(buf:bytes|bytearray, bit_position: int) -> tuple[int, int]:
|
||||||
nb_zeroes=0
|
nb_zeroes=0
|
||||||
while True:
|
while True:
|
||||||
bit_position, b = read_bit(buf, bit_position)
|
bit_position, b = read_bit(buf, bit_position)
|
||||||
@@ -97,7 +98,7 @@ def read_unsigned_exp_golomb(buf:bytes, bit_position: int) -> tuple[int, int]:
|
|||||||
return bit_position, v-1
|
return bit_position, v-1
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def read_signed_exp_golomb(buf:bytes, bit_position: int) -> tuple[int, int]:
|
def read_signed_exp_golomb(buf:bytes|bytearray, bit_position: int) -> tuple[int, int]:
|
||||||
bit_position, v = read_unsigned_exp_golomb(buf, bit_position)
|
bit_position, v = read_unsigned_exp_golomb(buf, bit_position)
|
||||||
match v%2:
|
match v%2:
|
||||||
case 0:
|
case 0:
|
||||||
@@ -172,7 +173,7 @@ def write_signed_exp_golomb(buf:bytearray, bit_position: int, v) -> int:
|
|||||||
return bit_position
|
return bit_position
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def parse_rbsp_trailing_bits(buf:bytes, bit_position: int) -> int:
|
def parse_rbsp_trailing_bits(buf:bytes|bytearray, bit_position: int) -> int:
|
||||||
bit_position, one = read_bit(buf, bit_position)
|
bit_position, one = read_bit(buf, bit_position)
|
||||||
if one==0:
|
if one==0:
|
||||||
raise ValueError(f'Stop bit should be equal to one. Read: {one:d}')
|
raise ValueError(f'Stop bit should be equal to one. Read: {one:d}')
|
||||||
@@ -192,7 +193,7 @@ def write_rbsp_trailing_bits(buf:bytearray, bit_position: int) -> int:
|
|||||||
return bit_position
|
return bit_position
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def more_rbsp_data(buf:bytes, bit_position: int) -> bool:
|
def more_rbsp_data(buf:bytes|bytearray, bit_position: int) -> bool:
|
||||||
logger.debug('Is there more data in buffer of length: %d at bit position: %d',
|
logger.debug('Is there more data in buffer of length: %d at bit position: %d',
|
||||||
len(buf), bit_position)
|
len(buf), bit_position)
|
||||||
|
|
||||||
@@ -249,7 +250,7 @@ def sodb_to_rbsp(buf:bytes) -> bytes:
|
|||||||
|
|
||||||
# Useful for SPS and PPS
|
# Useful for SPS and PPS
|
||||||
@typechecked
|
@typechecked
|
||||||
def parse_scaling_list(buf:bytes, bit_position: int, size) -> tuple[int,list[int]]:
|
def parse_scaling_list(buf:bytes|bytearray, bit_position: int, size) -> tuple[int,list[int]]:
|
||||||
res = []
|
res = []
|
||||||
last_scale = 8
|
last_scale = 8
|
||||||
next_scale = 8
|
next_scale = 8
|
||||||
@@ -284,7 +285,7 @@ def write_scaling_list(buf:bytearray, bit_position: int, size, matrix:list[int],
|
|||||||
bit_position = write_signed_exp_golomb(buf, bit_position, delta)
|
bit_position = write_signed_exp_golomb(buf, bit_position, delta)
|
||||||
else:
|
else:
|
||||||
logger.error('Not yet implemented')
|
logger.error('Not yet implemented')
|
||||||
exit(-1)
|
raise UnimplementedFeatureError("Optimized scaling list of H264 header is not implemented")
|
||||||
# reverse = deltas.reverse()
|
# reverse = deltas.reverse()
|
||||||
# compressed = False
|
# compressed = False
|
||||||
# while len(reverse)>0:
|
# while len(reverse)>0:
|
||||||
|
|||||||
@@ -4,12 +4,14 @@
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from math import floor, log, ceil
|
from math import ceil, floor, log
|
||||||
|
|
||||||
from typeguard import typechecked
|
|
||||||
import hexdump
|
import hexdump
|
||||||
|
|
||||||
from tscut.h264.bitstream import (
|
from tscut.h264.bitstream import (
|
||||||
|
more_rbsp_data,
|
||||||
|
parse_rbsp_trailing_bits,
|
||||||
|
parse_scaling_list,
|
||||||
read_bit,
|
read_bit,
|
||||||
read_bits,
|
read_bits,
|
||||||
read_boolean,
|
read_boolean,
|
||||||
@@ -28,9 +30,6 @@ from tscut.h264.bitstream import (
|
|||||||
write_signed_exp_golomb,
|
write_signed_exp_golomb,
|
||||||
write_unsigned_exp_golomb,
|
write_unsigned_exp_golomb,
|
||||||
write_word,
|
write_word,
|
||||||
parse_scaling_list,
|
|
||||||
parse_rbsp_trailing_bits,
|
|
||||||
more_rbsp_data
|
|
||||||
)
|
)
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|||||||
@@ -6,20 +6,18 @@
|
|||||||
import logging
|
import logging
|
||||||
from typing import IO
|
from typing import IO
|
||||||
|
|
||||||
from typeguard import typechecked
|
|
||||||
import hexdump
|
import hexdump
|
||||||
|
from typeguard import typechecked
|
||||||
|
|
||||||
|
from tscut.exceptions import InvalidMediaError
|
||||||
|
from tscut.h264.avc import AVCDecoderConfiguration, get_avc_config_from_h264
|
||||||
from tscut.matroska.ebml import get_ebml_length
|
from tscut.matroska.ebml import get_ebml_length
|
||||||
from tscut.h264.avc import (
|
|
||||||
AVCDecoderConfiguration,
|
|
||||||
get_avc_config_from_h264
|
|
||||||
)
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def dump_codec_private_data(avc_decoder_configuration: AVCDecoderConfiguration) -> bytearray | None:
|
def dump_codec_private_data(avc_decoder_configuration: AVCDecoderConfiguration) -> bytearray:
|
||||||
# Rebuild a Matroska Codec Private Element
|
# Rebuild a Matroska Codec Private Element
|
||||||
res = bytearray()
|
res = bytearray()
|
||||||
# Code private element
|
# Code private element
|
||||||
@@ -30,7 +28,7 @@ def dump_codec_private_data(avc_decoder_configuration: AVCDecoderConfiguration)
|
|||||||
|
|
||||||
embl_length = get_ebml_length(len(buf))
|
embl_length = get_ebml_length(len(buf))
|
||||||
if embl_length is None:
|
if embl_length is None:
|
||||||
return None
|
raise InvalidMediaError
|
||||||
logger.debug('EMBL encoded length: %s', hexdump.dump(embl_length, sep=':'))
|
logger.debug('EMBL encoded length: %s', hexdump.dump(embl_length, sep=':'))
|
||||||
res.extend(embl_length)
|
res.extend(embl_length)
|
||||||
res.extend(buf)
|
res.extend(buf)
|
||||||
@@ -38,7 +36,7 @@ def dump_codec_private_data(avc_decoder_configuration: AVCDecoderConfiguration)
|
|||||||
return res
|
return res
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def get_codec_private_data_from_h264(input_file: IO[bytes]) -> bytearray | None:
|
def get_codec_private_data_from_h264(input_file: IO[bytes]) -> bytearray:
|
||||||
avcconfig = get_avc_config_from_h264(input_file)
|
avcconfig = get_avc_config_from_h264(input_file)
|
||||||
res = dump_codec_private_data(avcconfig)
|
res = dump_codec_private_data(avcconfig)
|
||||||
|
|
||||||
|
|||||||
@@ -13,8 +13,8 @@ from os import (
|
|||||||
)
|
)
|
||||||
from typing import IO
|
from typing import IO
|
||||||
|
|
||||||
from typeguard import typechecked
|
|
||||||
import hexdump
|
import hexdump
|
||||||
|
from typeguard import typechecked
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
#
|
||||||
|
# Copyright (C) 2026 Frédéric Tronel
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import timedelta
|
||||||
|
from enum import IntEnum, unique
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import IO, BinaryIO
|
||||||
|
|
||||||
|
from tscut.h264.avc import AVCDecoderConfiguration
|
||||||
|
|
||||||
|
|
||||||
|
@unique
|
||||||
|
class SupportedFormat(IntEnum):
|
||||||
|
TS = 1
|
||||||
|
MP4 = 2
|
||||||
|
MATROSKA = 3
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
match self:
|
||||||
|
case SupportedFormat.TS:
|
||||||
|
return 'mpegts'
|
||||||
|
case SupportedFormat.MP4:
|
||||||
|
return 'mov,mp4,m4a,3gp,3g2,mj2'
|
||||||
|
case SupportedFormat.MATROSKA:
|
||||||
|
return 'matroska,webm'
|
||||||
|
case _:
|
||||||
|
return 'Unsupported format'
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ProcessingOptions:
|
||||||
|
input_file: Path
|
||||||
|
output_file: Path
|
||||||
|
parts: list[tuple[timedelta, timedelta]]
|
||||||
|
tools_paths: dict[str, str]
|
||||||
|
all_optional_tools: bool
|
||||||
|
framerate: int|None = None
|
||||||
|
threshold: int = 0
|
||||||
|
coarse: bool = False
|
||||||
|
subtitles_ocr: bool = True
|
||||||
|
verbose: bool = False
|
||||||
|
dump_memory: bool = False
|
||||||
|
keep_temporaries: bool = False
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class PreparedMedia:
|
||||||
|
basename: str
|
||||||
|
movie: BinaryIO
|
||||||
|
duration: timedelta
|
||||||
|
framerate: float
|
||||||
|
streams: list
|
||||||
|
width: int
|
||||||
|
height: int
|
||||||
|
avc_config: AVCDecoderConfiguration
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class CutResult:
|
||||||
|
filename: str
|
||||||
|
movie: IO[bytes]
|
||||||
|
check_positions: list[timedelta]
|
||||||
@@ -0,0 +1,253 @@
|
|||||||
|
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
#
|
||||||
|
# Copyright (C) 2026 Frédéric Tronel
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from shutil import copyfile, move
|
||||||
|
from typing import BinaryIO
|
||||||
|
|
||||||
|
import hexdump
|
||||||
|
|
||||||
|
from tscut.cutting import cut_recording
|
||||||
|
from tscut.exceptions import InvalidMediaError, TemporaryFileError
|
||||||
|
from tscut.h264.avc import parse_codec_private
|
||||||
|
from tscut.matroska.codec import dump_codec_private_data
|
||||||
|
from tscut.models import PreparedMedia, ProcessingOptions, SupportedFormat
|
||||||
|
from tscut.subtitles.ocr import do_ocr, extract_srt, get_tesseract_supported_lang
|
||||||
|
from tscut.temporaries import TemporaryFiles
|
||||||
|
from tscut.tools.ffmpeg import ffmpeg_convert
|
||||||
|
from tscut.tools.ffprobe import (
|
||||||
|
find_subtitles_tracks,
|
||||||
|
get_format,
|
||||||
|
get_frame_rate,
|
||||||
|
get_movie_duration,
|
||||||
|
get_streams,
|
||||||
|
)
|
||||||
|
from tscut.tools.mkvtoolnix import (
|
||||||
|
get_codec_private_data_from_mkv,
|
||||||
|
remux_srt_subtitles,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def detect_supported_format(format_name: str) -> SupportedFormat:
|
||||||
|
for media_format in SupportedFormat:
|
||||||
|
if format_name == str(media_format):
|
||||||
|
return media_format
|
||||||
|
|
||||||
|
raise InvalidMediaError(
|
||||||
|
f"Unsupported media format: {format_name}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def process_recording(options: ProcessingOptions, temporaries: TemporaryFiles) -> None:
|
||||||
|
nb_parts = len(options.parts)
|
||||||
|
basename = os.path.splitext(os.path.basename(options.input_file))[0]
|
||||||
|
mp4filename = basename+'.mp4'
|
||||||
|
mkvfilename = basename+'.mkv'
|
||||||
|
|
||||||
|
try:
|
||||||
|
input_file = open(options.input_file, mode='rb')
|
||||||
|
logger.debug("Type of input file: %s", type(input_file))
|
||||||
|
except OSError as e:
|
||||||
|
raise InvalidMediaError(f"Impossible to open {options.input_file}") from e
|
||||||
|
|
||||||
|
format_of_file = get_format(options.tools_paths['ffprobe'], input_file)
|
||||||
|
|
||||||
|
if format_of_file is None:
|
||||||
|
raise InvalidMediaError(f"Impossible to retrieve format of file: {input_file}")
|
||||||
|
if 'format_name' not in format_of_file:
|
||||||
|
raise InvalidMediaError(f"Impossible to retrieve format of file: {input_file}")
|
||||||
|
if 'duration' not in format_of_file:
|
||||||
|
raise InvalidMediaError(f"Impossible to retrieve duration of file: {input_file}")
|
||||||
|
format_name = format_of_file['format_name']
|
||||||
|
|
||||||
|
duration = timedelta(seconds=float(format_of_file['duration']))
|
||||||
|
logger.info("Durée de l'enregistrement: %s", duration)
|
||||||
|
|
||||||
|
if options.framerate is None:
|
||||||
|
framerate = get_frame_rate(options.tools_paths['ffprobe'], input_file)
|
||||||
|
if framerate is None:
|
||||||
|
raise InvalidMediaError('Impossible to estimate frame rate !')
|
||||||
|
else:
|
||||||
|
framerate = options.framerate
|
||||||
|
|
||||||
|
logger.info('Frame rate: %.1f fps', framerate)
|
||||||
|
|
||||||
|
final_format_of_file = detect_supported_format(format_name)
|
||||||
|
|
||||||
|
mkv: BinaryIO
|
||||||
|
|
||||||
|
if final_format_of_file == SupportedFormat.TS:
|
||||||
|
logger.info("Converting TS to MP4 (to fix timestamps).")
|
||||||
|
try:
|
||||||
|
with open(mp4filename, 'wb+') as mp4:
|
||||||
|
ffmpeg_convert(options.tools_paths['ffmpeg'], options.tools_paths['ffprobe'],
|
||||||
|
input_file, 'mpegts', mp4, 'mp4', duration)
|
||||||
|
temporaries.add(mp4)
|
||||||
|
logger.info("Converting MP4 to MKV.")
|
||||||
|
try:
|
||||||
|
mkv = open(mkvfilename, 'wb+')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to create: {mkvfilename}") from e
|
||||||
|
|
||||||
|
ffmpeg_convert(options.tools_paths['ffmpeg'], options.tools_paths['ffprobe'],
|
||||||
|
mp4, 'mp4', mkv, 'matroska', duration)
|
||||||
|
if nb_parts > 0:
|
||||||
|
temporaries.add(mkv)
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to create: {mp4filename}") from e
|
||||||
|
|
||||||
|
|
||||||
|
elif final_format_of_file == SupportedFormat.MP4:
|
||||||
|
logger.info("Converting MP4 to MKV")
|
||||||
|
try:
|
||||||
|
mkv = open(mkvfilename, 'wb+')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to create: {mkvfilename}") from e
|
||||||
|
ffmpeg_convert(options.tools_paths['ffmpeg'], options.tools_paths['ffprobe'],
|
||||||
|
input_file, 'mp4', mkv, 'matroska', duration)
|
||||||
|
if nb_parts > 0:
|
||||||
|
temporaries.add(mkv)
|
||||||
|
else:
|
||||||
|
logger.info("Already in MKV")
|
||||||
|
mkv = input_file
|
||||||
|
|
||||||
|
streams = get_streams(options.tools_paths['ffprobe'], mkv)
|
||||||
|
if streams is None:
|
||||||
|
raise InvalidMediaError(f"No streams found in file: {mkv}")
|
||||||
|
|
||||||
|
logger.debug('Streams: %s', streams)
|
||||||
|
main_video = None
|
||||||
|
nb_videos = 0
|
||||||
|
for stream in streams:
|
||||||
|
if stream['codec_type'] == 'video':
|
||||||
|
if stream['disposition']['default'] == 1:
|
||||||
|
main_video = stream
|
||||||
|
width = stream['width']
|
||||||
|
height = stream['height']
|
||||||
|
break
|
||||||
|
nb_videos+=1
|
||||||
|
if nb_videos == 1:
|
||||||
|
main_video = stream
|
||||||
|
width = stream['width']
|
||||||
|
height = stream['height']
|
||||||
|
else:
|
||||||
|
main_video = None
|
||||||
|
|
||||||
|
if main_video is None:
|
||||||
|
raise InvalidMediaError("Impossible to find main video stream.")
|
||||||
|
|
||||||
|
# We retrieve the main private codec data
|
||||||
|
_, main_codec_private_data = get_codec_private_data_from_mkv(
|
||||||
|
mkvinfo_path=options.tools_paths['mkvinfo'],
|
||||||
|
input_file=mkv
|
||||||
|
)
|
||||||
|
logger.debug('Main video stream has following private data: %s',
|
||||||
|
hexdump.dump(main_codec_private_data, sep=':'))
|
||||||
|
|
||||||
|
# We parse them
|
||||||
|
main_avc_config = parse_codec_private(main_codec_private_data)
|
||||||
|
logger.debug('AVC configuration: %s', main_avc_config)
|
||||||
|
|
||||||
|
# We check if the parse and dump operations are idempotent.
|
||||||
|
private_data = dump_codec_private_data(main_avc_config)
|
||||||
|
logger.debug('Redump AVC configuration: %s', hexdump.dump(private_data, sep=':'))
|
||||||
|
# In rare occasion, the PPS has trailing zeroes that do not seem to be related to useful data
|
||||||
|
# but they differ from the private data we generate that do not contain them.
|
||||||
|
# In that case we try to redecode our own private data to see if both AVC configurations are
|
||||||
|
# the same.
|
||||||
|
if main_codec_private_data != private_data:
|
||||||
|
logger.warning('Difference detected in bitstream !!')
|
||||||
|
iso_avc_config = parse_codec_private(private_data)
|
||||||
|
logger.debug('Reread AVC configuration: %s', iso_avc_config)
|
||||||
|
|
||||||
|
# If there exists a difference between our own reconstructed AVC configuration and the
|
||||||
|
# original one, we abandon
|
||||||
|
if iso_avc_config != main_avc_config:
|
||||||
|
raise InvalidMediaError(f"AVC configurations are different:\
|
||||||
|
{main_avc_config}\n{iso_avc_config}\n")
|
||||||
|
|
||||||
|
prepared_movie = PreparedMedia(
|
||||||
|
basename = basename,
|
||||||
|
movie = mkv,
|
||||||
|
duration = duration,
|
||||||
|
framerate = framerate,
|
||||||
|
streams = streams,
|
||||||
|
width = int(width),
|
||||||
|
height = int(height),
|
||||||
|
avc_config = main_avc_config
|
||||||
|
)
|
||||||
|
|
||||||
|
cut_movie = cut_recording(prepared_movie, options, temporaries)
|
||||||
|
|
||||||
|
final_with_video_name = cut_movie.filename
|
||||||
|
final_with_video = cut_movie.movie
|
||||||
|
checks = cut_movie.check_positions
|
||||||
|
|
||||||
|
if options.subtitles_ocr:
|
||||||
|
if not options.all_optional_tools:
|
||||||
|
logger.warning("Missing tools for extracting subtitles.")
|
||||||
|
move(final_with_video_name, options.output_file)
|
||||||
|
else:
|
||||||
|
# Final cut is not any more the final step.
|
||||||
|
temporaries.add(final_with_video)
|
||||||
|
duration = get_movie_duration(options.tools_paths['ffprobe'], final_with_video)
|
||||||
|
supported_langs = get_tesseract_supported_lang(options.tools_paths['tesseract'])
|
||||||
|
logger.info('Supported lang: %s', supported_langs)
|
||||||
|
logger.info('Find subtitles tracks and language.')
|
||||||
|
subtitles = find_subtitles_tracks(options.tools_paths['ffprobe'], final_with_video)
|
||||||
|
logger.info(subtitles)
|
||||||
|
sts: dict[str, list[int]] = {}
|
||||||
|
for subtitle in subtitles:
|
||||||
|
index = subtitle['index']
|
||||||
|
if 'tags' in subtitle:
|
||||||
|
if 'language' in subtitle['tags']:
|
||||||
|
lang = subtitle['tags']['language']
|
||||||
|
if lang in sts:
|
||||||
|
sts[lang].append(index)
|
||||||
|
else:
|
||||||
|
sts[lang] = [index]
|
||||||
|
else:
|
||||||
|
logger.error("Dropping subtitle: %s because it is missing language\
|
||||||
|
indication", subtitle)
|
||||||
|
else:
|
||||||
|
logger.error("Dropping subtitle: %s because it is missing language indication",
|
||||||
|
subtitle)
|
||||||
|
|
||||||
|
logger.info(sts)
|
||||||
|
if len(sts) > 0:
|
||||||
|
logger.info('Supported languages: %s', supported_langs)
|
||||||
|
list_of_subtitles = extract_srt(options.tools_paths['mkvextract'],
|
||||||
|
final_with_video_name, sts, supported_langs)
|
||||||
|
logger.info(list_of_subtitles)
|
||||||
|
for idx_name, sub_name, _, _ in list_of_subtitles:
|
||||||
|
try:
|
||||||
|
idx = open(idx_name,'rb')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to open {idx_name}") from e
|
||||||
|
try:
|
||||||
|
sub = open(sub_name,'rb')
|
||||||
|
except OSError as e:
|
||||||
|
raise TemporaryFileError(f"Impossible to open {sub_name}") from e
|
||||||
|
|
||||||
|
temporaries.add(idx)
|
||||||
|
temporaries.add(sub)
|
||||||
|
|
||||||
|
ocr = do_ocr(options.tools_paths['vobsubocr'], list_of_subtitles, duration,
|
||||||
|
temporaries, options.dump_memory)
|
||||||
|
logger.info(ocr)
|
||||||
|
|
||||||
|
# Remux SRT subtitles
|
||||||
|
remux_srt_subtitles(options.tools_paths['mkvmerge'], final_with_video,
|
||||||
|
options.output_file, ocr)
|
||||||
|
else:
|
||||||
|
copyfile(final_with_video_name, options.output_file)
|
||||||
|
else:
|
||||||
|
move(final_with_video_name, options.output_file)
|
||||||
|
|
||||||
|
d = datetime(1,1,1)
|
||||||
|
for c in checks:
|
||||||
|
logger.info("Please check cut smoothness at %s", (c+d).strftime("%H:%M:%S"))
|
||||||
@@ -4,30 +4,33 @@
|
|||||||
|
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from typing import IO
|
import os
|
||||||
from io import TextIOWrapper
|
import re
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
from io import TextIOWrapper
|
||||||
from os import (
|
from os import (
|
||||||
read,
|
|
||||||
SEEK_SET,
|
SEEK_SET,
|
||||||
|
fstat,
|
||||||
lseek,
|
lseek,
|
||||||
memfd_create,
|
memfd_create,
|
||||||
|
read,
|
||||||
write,
|
write,
|
||||||
fstat,
|
|
||||||
)
|
)
|
||||||
import re
|
|
||||||
import os
|
|
||||||
from subprocess import PIPE, Popen
|
from subprocess import PIPE, Popen
|
||||||
|
from typing import Sequence
|
||||||
|
|
||||||
from iso639 import Lang
|
from iso639 import Lang
|
||||||
from iso639.exceptions import InvalidLanguageValue
|
from iso639.exceptions import InvalidLanguageValue
|
||||||
from typeguard import typechecked
|
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
|
from typeguard import typechecked
|
||||||
|
|
||||||
|
from tscut.exceptions import ExternalToolError
|
||||||
|
from tscut.temporaries import TemporaryFiles
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def get_tesseract_supported_lang(tesseract_path:str) -> dict[Lang, str]|None:
|
def get_tesseract_supported_lang(tesseract_path:str) -> dict[Lang, str]:
|
||||||
"""
|
"""
|
||||||
Retrieves the set of natural languages supported by the Tesseract OCR tool.
|
Retrieves the set of natural languages supported by the Tesseract OCR tool.
|
||||||
|
|
||||||
@@ -46,6 +49,7 @@ def get_tesseract_supported_lang(tesseract_path:str) -> dict[Lang, str]|None:
|
|||||||
res = {}
|
res = {}
|
||||||
|
|
||||||
with Popen([tesseract_path, '--list-langs'], stdout=PIPE) as tesseract:
|
with Popen([tesseract_path, '--list-langs'], stdout=PIPE) as tesseract:
|
||||||
|
assert tesseract.stdout is not None
|
||||||
for line in tesseract.stdout:
|
for line in tesseract.stdout:
|
||||||
line = line.decode('utf8')
|
line = line.decode('utf8')
|
||||||
p = re.compile('(?P<lang>[a-z]{3})\n')
|
p = re.compile('(?P<lang>[a-z]{3})\n')
|
||||||
@@ -61,8 +65,7 @@ def get_tesseract_supported_lang(tesseract_path:str) -> dict[Lang, str]|None:
|
|||||||
tesseract.wait()
|
tesseract.wait()
|
||||||
|
|
||||||
if tesseract.returncode != 0:
|
if tesseract.returncode != 0:
|
||||||
logger.error("Tesseract returns an error code: %d",tesseract.returncode)
|
raise ExternalToolError("Tesseract returns an error code: %d",tesseract.returncode)
|
||||||
return None
|
|
||||||
|
|
||||||
return res
|
return res
|
||||||
|
|
||||||
@@ -70,7 +73,7 @@ def get_tesseract_supported_lang(tesseract_path:str) -> dict[Lang, str]|None:
|
|||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def extract_srt(mkvextract:str, filename:str, subtitles:dict[str, list[int]],
|
def extract_srt(mkvextract:str, filename:str, subtitles:dict[str, list[int]],
|
||||||
langs:dict[Lang,str]) -> list[tuple[str,str,str,str]]|None:
|
langs:dict[Lang,str]) -> list[tuple[str,str,str,str]]:
|
||||||
params = [mkvextract, filename, 'tracks']
|
params = [mkvextract, filename, 'tracks']
|
||||||
|
|
||||||
res = []
|
res = []
|
||||||
@@ -98,6 +101,7 @@ def extract_srt(mkvextract:str, filename:str, subtitles:dict[str, list[int]],
|
|||||||
|
|
||||||
env = {**os.environ, 'LANG': 'C'}
|
env = {**os.environ, 'LANG': 'C'}
|
||||||
with Popen(params, stdout=PIPE, close_fds=False, env=env) as extract:
|
with Popen(params, stdout=PIPE, close_fds=False, env=env) as extract:
|
||||||
|
assert extract.stdout is not None
|
||||||
pb = tqdm(TextIOWrapper(extract.stdout, encoding="utf-8"), total=100, unit='%',
|
pb = tqdm(TextIOWrapper(extract.stdout, encoding="utf-8"), total=100, unit='%',
|
||||||
desc='Extraction:')
|
desc='Extraction:')
|
||||||
for line in pb:
|
for line in pb:
|
||||||
@@ -106,6 +110,7 @@ def extract_srt(mkvextract:str, filename:str, subtitles:dict[str, list[int]],
|
|||||||
m = p.match(line)
|
m = p.match(line)
|
||||||
if m is None:
|
if m is None:
|
||||||
logger.error('Impossible to parse progress')
|
logger.error('Impossible to parse progress')
|
||||||
|
else:
|
||||||
pb.update(int(m['progress'])-pb.n)
|
pb.update(int(m['progress'])-pb.n)
|
||||||
pb.update(100-pb.n)
|
pb.update(100-pb.n)
|
||||||
pb.refresh()
|
pb.refresh()
|
||||||
@@ -120,14 +125,13 @@ def extract_srt(mkvextract:str, filename:str, subtitles:dict[str, list[int]],
|
|||||||
case 1:
|
case 1:
|
||||||
logger.warning('Mkvextract returns warning')
|
logger.warning('Mkvextract returns warning')
|
||||||
case 2:
|
case 2:
|
||||||
logger.error('Mkvextract returns an error code: %d', extract.returncode)
|
raise ExternalToolError('Mkvextract returns an error code: %d', extract.returncode)
|
||||||
res = None
|
|
||||||
|
|
||||||
return res
|
return res
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def do_ocr(vobsubocr:str, idxs: list[tuple[str,str,str,str]], duration:timedelta,
|
def do_ocr(vobsubocr:str, idxs: Sequence[tuple[str,str,str,str]], duration:timedelta,
|
||||||
temporaries:list[IO[bytes]], dump_mem_fd:bool=False):
|
temporaries:TemporaryFiles, dump_mem_fd:bool=False):
|
||||||
res = []
|
res = []
|
||||||
|
|
||||||
for idx_name, _, lang, iso in idxs:
|
for idx_name, _, lang, iso in idxs:
|
||||||
@@ -140,6 +144,7 @@ def do_ocr(vobsubocr:str, idxs: list[tuple[str,str,str,str]], duration:timedelta
|
|||||||
|
|
||||||
srtfd = memfd_create(srtname, flags=0)
|
srtfd = memfd_create(srtname, flags=0)
|
||||||
with Popen([vobsubocr, '--lang', iso, idx_name], stdout=PIPE) as ocr:
|
with Popen([vobsubocr, '--lang', iso, idx_name], stdout=PIPE) as ocr:
|
||||||
|
assert ocr.stdout is not None
|
||||||
pb = tqdm(TextIOWrapper(ocr.stdout, encoding="utf-8"), total=
|
pb = tqdm(TextIOWrapper(ocr.stdout, encoding="utf-8"), total=
|
||||||
int(duration/timedelta(seconds=1)), unit='s', desc='OCR')
|
int(duration/timedelta(seconds=1)), unit='s', desc='OCR')
|
||||||
for line in pb:
|
for line in pb:
|
||||||
@@ -152,7 +157,7 @@ def do_ocr(vobsubocr:str, idxs: list[tuple[str,str,str,str]], duration:timedelta
|
|||||||
m = re.match(timestamps, line)
|
m = re.match(timestamps, line)
|
||||||
if m is not None:
|
if m is not None:
|
||||||
hours = int(m.group('hours'))
|
hours = int(m.group('hours'))
|
||||||
minutes = int(m.group('hours'))
|
minutes = int(m.group('minutes'))
|
||||||
seconds = int(m.group('seconds'))
|
seconds = int(m.group('seconds'))
|
||||||
ts = timedelta(hours=hours, minutes=minutes, seconds=seconds)
|
ts = timedelta(hours=hours, minutes=minutes, seconds=seconds)
|
||||||
pb.n = int(ts/timedelta(seconds=1))
|
pb.n = int(ts/timedelta(seconds=1))
|
||||||
@@ -173,7 +178,7 @@ def do_ocr(vobsubocr:str, idxs: list[tuple[str,str,str,str]], duration:timedelta
|
|||||||
pos = 0
|
pos = 0
|
||||||
while pos < srt_length:
|
while pos < srt_length:
|
||||||
pos+=write(outfd, buf[pos:])
|
pos+=write(outfd, buf[pos:])
|
||||||
temporaries.append(dump_srt)
|
temporaries.add(dump_srt)
|
||||||
except OSError:
|
except OSError:
|
||||||
logger.error('Impossible to create file: %s', srtname)
|
logger.error('Impossible to create file: %s', srtname)
|
||||||
return None
|
return None
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
# SPDX-License-Identifier: GPL-2.0-or-later
|
||||||
|
#
|
||||||
|
# Copyright (C) 2026 Frédéric Tronel
|
||||||
|
|
||||||
|
"""A class to manage (and clean) all temporary files created during conversion"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from typing import IO, Any, Self
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
class TemporaryFiles:
|
||||||
|
def __init__(self, keep: bool = False) -> None:
|
||||||
|
self._files: list[IO[Any]] = []
|
||||||
|
self._keep = keep
|
||||||
|
|
||||||
|
def add(self, file: IO[Any]) -> None:
|
||||||
|
self._files.append(file)
|
||||||
|
|
||||||
|
def cleanup(self) -> None:
|
||||||
|
if self._keep:
|
||||||
|
return
|
||||||
|
|
||||||
|
for f in self._files:
|
||||||
|
path = os.path.realpath(f.name)
|
||||||
|
logger.info("Removing: %s", path)
|
||||||
|
try:
|
||||||
|
f.close()
|
||||||
|
os.unlink(path)
|
||||||
|
except OSError:
|
||||||
|
logger.exception("Unable to remove temporary file: %s", path)
|
||||||
|
|
||||||
|
def __enter__(self) -> Self:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self) -> None:
|
||||||
|
self.cleanup()
|
||||||
+45
-75
@@ -3,38 +3,32 @@
|
|||||||
# Copyright (C) 2026 Frédéric Tronel
|
# Copyright (C) 2026 Frédéric Tronel
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from math import floor, ceil, log
|
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from os import (
|
|
||||||
read,
|
|
||||||
SEEK_SET,
|
|
||||||
lseek,
|
|
||||||
memfd_create,
|
|
||||||
set_inheritable,
|
|
||||||
write,
|
|
||||||
close
|
|
||||||
)
|
|
||||||
from typing import IO
|
|
||||||
from subprocess import PIPE, Popen
|
|
||||||
from io import TextIOWrapper
|
from io import TextIOWrapper
|
||||||
|
from math import ceil, floor, log
|
||||||
|
from os import SEEK_SET, close, lseek, memfd_create, read, set_inheritable, write
|
||||||
|
from subprocess import PIPE, Popen
|
||||||
|
from typing import IO, BinaryIO
|
||||||
|
|
||||||
from typeguard import typechecked
|
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
|
from typeguard import typechecked
|
||||||
|
|
||||||
|
from tscut.exceptions import ExternalToolError, InvalidMediaError, TemporaryFileError
|
||||||
|
from tscut.temporaries import TemporaryFiles
|
||||||
from tscut.tools.ffprobe import (
|
from tscut.tools.ffprobe import (
|
||||||
|
get_frames_in_stream,
|
||||||
get_video_dimensions,
|
get_video_dimensions,
|
||||||
with_subtitles,
|
with_subtitles,
|
||||||
get_frames_in_stream,
|
|
||||||
)
|
)
|
||||||
from tscut.tools.ppm import dump_ppm
|
from tscut.tools.ppm import dump_ppm
|
||||||
from tscut.tools.timeframe import parse_timestamp, get_packet_duration
|
from tscut.tools.timeframe import get_packet_duration, parse_timestamp
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def ffmpeg_convert(ffmpeg_path:str, ffprobe_path:str, input_file: IO[bytes], input_format:str,
|
def ffmpeg_convert(ffmpeg_path:str, ffprobe_path:str, input_file: IO[bytes], input_format:str,
|
||||||
output_file: IO[bytes], output_format:str, duration: timedelta):
|
output_file: IO[bytes], output_format:str, duration: timedelta) -> None:
|
||||||
width, height = get_video_dimensions(ffprobe_path, input_file)
|
width, height = get_video_dimensions(ffprobe_path, input_file)
|
||||||
if width is None or height is None:
|
if width is None or height is None:
|
||||||
return
|
return
|
||||||
@@ -46,10 +40,7 @@ def ffmpeg_convert(ffmpeg_path:str, ffprobe_path:str, input_file: IO[bytes], inp
|
|||||||
set_inheritable(infd, True)
|
set_inheritable(infd, True)
|
||||||
set_inheritable(outfd, True)
|
set_inheritable(outfd, True)
|
||||||
|
|
||||||
if logger.getEffectiveLevel() == logging.DEBUG:
|
log_level = [] if logger.getEffectiveLevel() == logging.DEBUG else ['-loglevel', 'quiet']
|
||||||
log_level = []
|
|
||||||
else:
|
|
||||||
log_level = [ '-loglevel', 'quiet' ]
|
|
||||||
|
|
||||||
params = [ffmpeg_path, '-y',]+log_level+['-progress', '/dev/stdout', '-canvas_size',
|
params = [ffmpeg_path, '-y',]+log_level+['-progress', '/dev/stdout', '-canvas_size',
|
||||||
f'{width:d}x{height:d}', '-f', input_format,
|
f'{width:d}x{height:d}', '-f', input_format,
|
||||||
@@ -66,6 +57,7 @@ def ffmpeg_convert(ffmpeg_path:str, ffprobe_path:str, input_file: IO[bytes], inp
|
|||||||
logger.debug('Executing %s', params)
|
logger.debug('Executing %s', params)
|
||||||
|
|
||||||
with Popen(params, stdout=PIPE, close_fds=False) as ffmpeg:
|
with Popen(params, stdout=PIPE, close_fds=False) as ffmpeg:
|
||||||
|
assert ffmpeg.stdout is not None
|
||||||
pb = tqdm(TextIOWrapper(ffmpeg.stdout, encoding="utf-8"),
|
pb = tqdm(TextIOWrapper(ffmpeg.stdout, encoding="utf-8"),
|
||||||
total=int(duration/timedelta(seconds=1)), unit='s', desc='Conversion')
|
total=int(duration/timedelta(seconds=1)), unit='s', desc='Conversion')
|
||||||
for line in pb:
|
for line in pb:
|
||||||
@@ -77,12 +69,12 @@ def ffmpeg_convert(ffmpeg_path:str, ffprobe_path:str, input_file: IO[bytes], inp
|
|||||||
pb.update()
|
pb.update()
|
||||||
status = ffmpeg.wait()
|
status = ffmpeg.wait()
|
||||||
if status != 0:
|
if status != 0:
|
||||||
logger.error('Conversion failed with status code: %d', status)
|
raise ExternalToolError(f"Conversion failed with status code: {status:d}")
|
||||||
|
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def extract_pictures(ffmpeg_path:str, input_file:IO[bytes], begin:timedelta, nb_frames:int,
|
def extract_pictures(ffmpeg_path:str, input_file:IO[bytes], begin:timedelta, nb_frames:int,
|
||||||
width:int=640, height:int=480) -> tuple[bytes,int]|tuple[None,None]:
|
width:int=640, height:int=480) -> tuple[bytes,int]:
|
||||||
"""
|
"""
|
||||||
Extract pictures from a video file using FFmpeg.
|
Extract pictures from a video file using FFmpeg.
|
||||||
|
|
||||||
@@ -125,14 +117,12 @@ def extract_pictures(ffmpeg_path:str, input_file:IO[bytes], begin:timedelta, nb_
|
|||||||
with Popen(command, stdout=PIPE, close_fds=False) as ffmpeg:
|
with Popen(command, stdout=PIPE, close_fds=False) as ffmpeg:
|
||||||
status = ffmpeg.wait()
|
status = ffmpeg.wait()
|
||||||
if status != 0:
|
if status != 0:
|
||||||
logger.error('Conversion failed with status code: %d', status)
|
raise ExternalToolError(f"Conversion failed with status code: {status:d}")
|
||||||
return None, None
|
|
||||||
|
|
||||||
lseek(outfd, 0, SEEK_SET)
|
lseek(outfd, 0, SEEK_SET)
|
||||||
images = read(outfd,length)
|
images = read(outfd,length)
|
||||||
if len(images) != length:
|
if len(images) != length:
|
||||||
logger.error("Received %d bytes but %d were expected.", len(images), length)
|
raise InvalidMediaError(f"Received {len(images)} bytes but {length} were expected.")
|
||||||
return None, None
|
|
||||||
|
|
||||||
lseek(outfd, 0, SEEK_SET)
|
lseek(outfd, 0, SEEK_SET)
|
||||||
return images, outfd
|
return images, outfd
|
||||||
@@ -141,7 +131,7 @@ def extract_pictures(ffmpeg_path:str, input_file:IO[bytes], begin:timedelta, nb_
|
|||||||
def extract_sound(ffmpeg_path:str, input_file: IO[bytes], begin:timedelta, output_filename:str,
|
def extract_sound(ffmpeg_path:str, input_file: IO[bytes], begin:timedelta, output_filename:str,
|
||||||
packet_duration:int, sub_channel:int=0,
|
packet_duration:int, sub_channel:int=0,
|
||||||
nb_packets:int=0, sample_rate:int=48000,
|
nb_packets:int=0, sample_rate:int=48000,
|
||||||
nb_channels:int=2) -> tuple[bytes,int]|tuple[None,None]:
|
nb_channels:int=2) -> tuple[bytes,int]:
|
||||||
outfd = memfd_create(output_filename, flags=0)
|
outfd = memfd_create(output_filename, flags=0)
|
||||||
infd = input_file.fileno()
|
infd = input_file.fileno()
|
||||||
lseek(infd, 0, SEEK_SET)
|
lseek(infd, 0, SEEK_SET)
|
||||||
@@ -159,24 +149,25 @@ def extract_sound(ffmpeg_path:str, input_file: IO[bytes], begin:timedelta, outpu
|
|||||||
with Popen(command, stdout=PIPE, close_fds=False) as ffmpeg:
|
with Popen(command, stdout=PIPE, close_fds=False) as ffmpeg:
|
||||||
status = ffmpeg.wait()
|
status = ffmpeg.wait()
|
||||||
if status != 0:
|
if status != 0:
|
||||||
logger.error('Sound extraction returns error code: %d', status)
|
raise ExternalToolError(f"Sound extraction returns error code: {status}")
|
||||||
return None, None
|
|
||||||
|
|
||||||
lseek(outfd, 0, SEEK_SET)
|
lseek(outfd, 0, SEEK_SET)
|
||||||
sound = read(outfd, length)
|
sound = read(outfd, length)
|
||||||
|
|
||||||
if len(sound) != length:
|
if len(sound) != length:
|
||||||
logger.info("Received %d bytes but %d were expected (channels=%d, freq=%d, packets=%d,\
|
raise InvalidMediaError(f"Received {len(sound)} bytes but {length} were expected (\
|
||||||
duration=%d ms).", len(sound), length, nb_channels, sample_rate, nb_packets,
|
channels={nb_channels}, freq={sample_rate} packets={nb_packets},\
|
||||||
packet_duration)
|
duration={packet_duration} ms).")
|
||||||
return None, None
|
|
||||||
|
|
||||||
return sound, outfd
|
return sound, outfd
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes], begin:timedelta,
|
def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes], begin:timedelta,
|
||||||
end:timedelta, streams, files_prefix, nb_frames:int, framerate:float,
|
end:timedelta, streams, files_prefix, nb_frames:int, framerate:float,
|
||||||
width:int, height:int, temporaries, dump_mem_fd:bool=False):
|
width:int, height:int, temporaries:TemporaryFiles,
|
||||||
|
dump_mem_fd:bool=False) -> tuple[BinaryIO|None,
|
||||||
|
TextIOWrapper|None,
|
||||||
|
BinaryIO|None]:
|
||||||
# The command line for encoding only video track
|
# The command line for encoding only video track
|
||||||
video_encoder_params = [ ffmpeg_path, '-y', '-loglevel', 'quiet']
|
video_encoder_params = [ ffmpeg_path, '-y', '-loglevel', 'quiet']
|
||||||
video_input_params = []
|
video_input_params = []
|
||||||
@@ -236,9 +227,6 @@ def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes],
|
|||||||
images_bytes, memfd = extract_pictures(ffmpeg_path, input_file=input_file,
|
images_bytes, memfd = extract_pictures(ffmpeg_path, input_file=input_file,
|
||||||
begin=begin, nb_frames=nb_frames,
|
begin=begin, nb_frames=nb_frames,
|
||||||
width=width, height=height)
|
width=width, height=height)
|
||||||
if images_bytes is None or memfd is None:
|
|
||||||
logger.error('Impossible to extract picture from video stream.')
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
memfds.append(memfd)
|
memfds.append(memfd)
|
||||||
if dump_mem_fd:
|
if dump_mem_fd:
|
||||||
@@ -253,7 +241,7 @@ def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes],
|
|||||||
video_codec_params.extend([f'-c:v:{video_id:d}', codec, f'-level:v:{video_id:d}',
|
video_codec_params.extend([f'-c:v:{video_id:d}', codec, f'-level:v:{video_id:d}',
|
||||||
level, '-pix_fmt', pixel_format])
|
level, '-pix_fmt', pixel_format])
|
||||||
video_codec_params.extend(interlaced_options)
|
video_codec_params.extend(interlaced_options)
|
||||||
video_codec_params.extend([f'-colorspace:v:{video_id}', color_space,
|
video_codec_params.extend([f'-colorspace:v:{video_id:d}', color_space,
|
||||||
f'-color_primaries:v:{video_id:d}', color_primaries,
|
f'-color_primaries:v:{video_id:d}', color_primaries,
|
||||||
f'-color_trc:v:{video_id:d}', color_transfer,
|
f'-color_trc:v:{video_id:d}', color_transfer,
|
||||||
f'-color_range:v:{video_id:d}', color_range])
|
f'-color_range:v:{video_id:d}', color_range])
|
||||||
@@ -262,26 +250,19 @@ def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes],
|
|||||||
logger.debug('Audio stream: %s', stream)
|
logger.debug('Audio stream: %s', stream)
|
||||||
sample_rate = int(stream['sample_rate'])
|
sample_rate = int(stream['sample_rate'])
|
||||||
nb_channels = int(stream['channels'])
|
nb_channels = int(stream['channels'])
|
||||||
if 'bit_rate' in stream:
|
bit_rate = int(stream['bit_rate']) if 'bit_rate' in stream else 128000
|
||||||
bit_rate = int(stream['bit_rate'])
|
|
||||||
else:
|
|
||||||
bit_rate = 128000
|
|
||||||
codec = stream['codec_name']
|
codec = stream['codec_name']
|
||||||
if 'tags' in stream:
|
if 'tags' in stream and 'language' in stream['tags']:
|
||||||
if 'language' in stream['tags']:
|
|
||||||
generic_codec_params.extend([f'-metadata:s:a:{audio_id:d}',
|
generic_codec_params.extend([f'-metadata:s:a:{audio_id:d}',
|
||||||
f"language={stream['tags']['language']}"])
|
f"language={stream['tags']['language']}"])
|
||||||
packets = get_frames_in_stream(ffprobe_path, input_file=input_file, begin=begin,
|
packets = get_frames_in_stream(ffprobe_path, input_file=input_file, begin=begin,
|
||||||
end=end, stream_kind='a', sub_stream_id=audio_id)
|
end=end, stream_kind='a', sub_stream_id=audio_id)
|
||||||
if packets is None:
|
if packets is None:
|
||||||
logger.error("Impossible to retrieve audio packets")
|
raise InvalidMediaError("Impossible to retrieve audio packets")
|
||||||
return None
|
|
||||||
nb_packets = len(packets)
|
nb_packets = len(packets)
|
||||||
logger.debug("Found %d packets to be extracted from audio track.", nb_packets)
|
logger.debug("Found %d packets to be extracted from audio track.", nb_packets)
|
||||||
if nb_packets > 0:
|
if nb_packets > 0:
|
||||||
packet_duration = get_packet_duration(packets[0])
|
packet_duration = get_packet_duration(packets[0])
|
||||||
if packet_duration is None:
|
|
||||||
return None
|
|
||||||
else:
|
else:
|
||||||
packet_duration = 0
|
packet_duration = 0
|
||||||
|
|
||||||
@@ -294,23 +275,18 @@ def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes],
|
|||||||
output_filename=tmpname,
|
output_filename=tmpname,
|
||||||
sample_rate=sample_rate, nb_channels=nb_channels)
|
sample_rate=sample_rate, nb_channels=nb_channels)
|
||||||
|
|
||||||
if sound_bytes is None or memfd is None:
|
|
||||||
logger.error('Impossible to extract sound track')
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
memfds.append(memfd)
|
memfds.append(memfd)
|
||||||
|
|
||||||
if dump_mem_fd:
|
if dump_mem_fd:
|
||||||
try:
|
try:
|
||||||
with open(tmpname,'wb') as output:
|
with open(tmpname,'wb') as output:
|
||||||
temporaries.append(output)
|
temporaries.add(output)
|
||||||
outfd = output.fileno()
|
outfd = output.fileno()
|
||||||
pos = 0
|
pos = 0
|
||||||
while pos < len(sound_bytes):
|
while pos < len(sound_bytes):
|
||||||
pos+=write(outfd, sound_bytes[pos:])
|
pos+=write(outfd, sound_bytes[pos:])
|
||||||
except OSError:
|
except OSError as e:
|
||||||
logger.error('Impossible to create file: %s', tmpname)
|
raise TemporaryFileError(f"Impossible to create file: {tmpname}") from e
|
||||||
return None
|
|
||||||
|
|
||||||
# We rewind to zero the memory file descriptor
|
# We rewind to zero the memory file descriptor
|
||||||
lseek(memfd, 0, SEEK_SET)
|
lseek(memfd, 0, SEEK_SET)
|
||||||
@@ -325,8 +301,7 @@ def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes],
|
|||||||
logger.info("Extracting a subtitle stream: s:%d", subtitle_id)
|
logger.info("Extracting a subtitle stream: s:%d", subtitle_id)
|
||||||
codec = stream['codec_name']
|
codec = stream['codec_name']
|
||||||
generic_input_params.extend(['-i', './empty.idx'])
|
generic_input_params.extend(['-i', './empty.idx'])
|
||||||
if 'tags' in stream:
|
if 'tags' in stream and 'language' in stream['tags']:
|
||||||
if 'language' in stream['tags']:
|
|
||||||
generic_codec_params.extend([f'-metadata:s:s:{subtitle_id:d}',
|
generic_codec_params.extend([f'-metadata:s:s:{subtitle_id:d}',
|
||||||
f"language={stream['tags']['language']}"])
|
f"language={stream['tags']['language']}"])
|
||||||
generic_codec_params.extend([f'-c:s:{subtitle_id:d}', 'copy'])
|
generic_codec_params.extend([f'-c:s:{subtitle_id:d}', 'copy'])
|
||||||
@@ -344,9 +319,8 @@ def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes],
|
|||||||
mkv_filename = f'{files_prefix}.mkv'
|
mkv_filename = f'{files_prefix}.mkv'
|
||||||
try:
|
try:
|
||||||
mkv_output = open(mkv_filename,'wb+')
|
mkv_output = open(mkv_filename,'wb+')
|
||||||
except OSError:
|
except OSError as e:
|
||||||
logger.error('Impossible to create file: %s', mkv_filename)
|
raise TemporaryFileError(f"Impossible to create file: {mkv_filename}") from e
|
||||||
return None
|
|
||||||
|
|
||||||
mkvoutfd = mkv_output.fileno()
|
mkvoutfd = mkv_output.fileno()
|
||||||
set_inheritable(mkvoutfd, True)
|
set_inheritable(mkvoutfd, True)
|
||||||
@@ -357,17 +331,15 @@ def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes],
|
|||||||
with Popen(generic_encoder_params, stdout=PIPE, close_fds=False) as ffmpeg:
|
with Popen(generic_encoder_params, stdout=PIPE, close_fds=False) as ffmpeg:
|
||||||
status = ffmpeg.wait()
|
status = ffmpeg.wait()
|
||||||
if status != 0:
|
if status != 0:
|
||||||
logger.error('Encoding failed with status code: %d', status)
|
raise ExternalToolError(f"Encoding failed with status code: {status}")
|
||||||
return None
|
|
||||||
|
|
||||||
temporaries.append(mkv_output)
|
temporaries.add(mkv_output)
|
||||||
|
|
||||||
h264_filename = f'{files_prefix}.h264'
|
h264_filename = f'{files_prefix}.h264'
|
||||||
try:
|
try:
|
||||||
h264_output = open(h264_filename,'wb+')
|
h264_output = open(h264_filename,'wb+')
|
||||||
except OSError:
|
except OSError as e:
|
||||||
logger.error('Impossible to create file: %s', h264_filename)
|
raise TemporaryFileError(f"Impossible to create file {h264_filename}") from e
|
||||||
return None
|
|
||||||
|
|
||||||
h264outfd = h264_output.fileno()
|
h264outfd = h264_output.fileno()
|
||||||
set_inheritable(h264outfd, True)
|
set_inheritable(h264outfd, True)
|
||||||
@@ -386,17 +358,15 @@ def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes],
|
|||||||
with Popen(video_encoder_params, stdout=PIPE, close_fds=False) as ffmpeg:
|
with Popen(video_encoder_params, stdout=PIPE, close_fds=False) as ffmpeg:
|
||||||
status = ffmpeg.wait()
|
status = ffmpeg.wait()
|
||||||
if status != 0:
|
if status != 0:
|
||||||
logger.error('Encoding failed with status code: %d', status)
|
raise ExternalToolError(f"Encoding failed with status code: {status:d}")
|
||||||
return None
|
|
||||||
|
|
||||||
temporaries.append(h264_output)
|
temporaries.add(h264_output)
|
||||||
|
|
||||||
h264_ts_filename = f'{files_prefix}-ts.txt'
|
h264_ts_filename = f'{files_prefix}-ts.txt'
|
||||||
try:
|
try:
|
||||||
h264_ts_output = open(h264_ts_filename,'w+', encoding='utf8')
|
h264_ts_output = open(h264_ts_filename,'w+', encoding='utf8')
|
||||||
except OSError:
|
except OSError as e:
|
||||||
logger.error('Impossible to create file: %s', h264_ts_filename)
|
raise TemporaryFileError(f"Impossible to create file: {h264_ts_filename}") from e
|
||||||
return None
|
|
||||||
|
|
||||||
h264_ts_output.write('# timestamp format v2\n')
|
h264_ts_output.write('# timestamp format v2\n')
|
||||||
ts = 0
|
ts = 0
|
||||||
@@ -406,7 +376,7 @@ def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes],
|
|||||||
h264_ts_output.flush()
|
h264_ts_output.flush()
|
||||||
h264_ts_output.seek(0)
|
h264_ts_output.seek(0)
|
||||||
|
|
||||||
temporaries.append(h264_ts_output)
|
temporaries.add(h264_ts_output)
|
||||||
|
|
||||||
for memfd in memfds:
|
for memfd in memfds:
|
||||||
close(memfd)
|
close(memfd)
|
||||||
@@ -414,5 +384,5 @@ def extract_all_streams(ffmpeg_path:str, ffprobe_path:str, input_file:IO[bytes],
|
|||||||
return h264_output, h264_ts_output, mkv_output
|
return h264_output, h264_ts_output, mkv_output
|
||||||
|
|
||||||
# Nothing to be done. We are already at a i-frame boundary.
|
# Nothing to be done. We are already at a i-frame boundary.
|
||||||
return None, None
|
return None, None, None
|
||||||
|
|
||||||
|
|||||||
+26
-42
@@ -4,20 +4,21 @@
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
from math import floor
|
|
||||||
from datetime import timedelta
|
|
||||||
import os
|
import os
|
||||||
|
from datetime import timedelta
|
||||||
|
from io import BytesIO
|
||||||
|
from math import floor
|
||||||
from os import (
|
from os import (
|
||||||
SEEK_SET,
|
SEEK_SET,
|
||||||
lseek,
|
lseek,
|
||||||
set_inheritable,
|
set_inheritable,
|
||||||
)
|
)
|
||||||
from typing import IO
|
|
||||||
from subprocess import PIPE, Popen
|
from subprocess import PIPE, Popen
|
||||||
from io import BytesIO
|
from typing import IO
|
||||||
|
|
||||||
from typeguard import typechecked
|
from typeguard import typechecked
|
||||||
|
|
||||||
|
from tscut.exceptions import InvalidMediaError, ExternalToolError
|
||||||
from tscut.tools.timeframe import get_ts_frame
|
from tscut.tools.timeframe import get_ts_frame
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -63,8 +64,7 @@ def get_frame_rate(ffprobe_path:str, input_file: IO[bytes]) -> float|None:
|
|||||||
out = json.load(BytesIO(out))
|
out = json.load(BytesIO(out))
|
||||||
if 'frames' in out:
|
if 'frames' in out:
|
||||||
for frame in out['frames']:
|
for frame in out['frames']:
|
||||||
if 'interlaced_frame' in frame:
|
if 'interlaced_frame' in frame and frame['interlaced_frame'] == 1:
|
||||||
if frame['interlaced_frame'] == 1:
|
|
||||||
interlaced = True
|
interlaced = True
|
||||||
if 'pts_time' in frame:
|
if 'pts_time' in frame:
|
||||||
ts = float(frame['pts_time'])
|
ts = float(frame['pts_time'])
|
||||||
@@ -155,7 +155,7 @@ def get_format(ffprobe_path:str, input_file: IO[bytes]) -> dict|None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def get_movie_duration(ffprobe_path:str, input_file: IO[bytes]) -> timedelta|None:
|
def get_movie_duration(ffprobe_path:str, input_file: IO[bytes]) -> timedelta:
|
||||||
infd = input_file.fileno()
|
infd = input_file.fileno()
|
||||||
lseek(infd, 0, SEEK_SET)
|
lseek(infd, 0, SEEK_SET)
|
||||||
set_inheritable(infd, True)
|
set_inheritable(infd, True)
|
||||||
@@ -165,16 +165,12 @@ def get_movie_duration(ffprobe_path:str, input_file: IO[bytes]) -> timedelta|Non
|
|||||||
out = json.load(BytesIO(out))
|
out = json.load(BytesIO(out))
|
||||||
if 'format' in out and 'duration' in out['format']:
|
if 'format' in out and 'duration' in out['format']:
|
||||||
duration = floor(float(out['format']['duration']))
|
duration = floor(float(out['format']['duration']))
|
||||||
ts = timedelta(seconds=duration)
|
return timedelta(seconds=duration)
|
||||||
return ts
|
raise InvalidMediaError("Impossible to retrieve duration of movie")
|
||||||
logger.error('Impossible to retrieve duration of movie')
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
# ffprobe -loglevel quiet -select_streams v:0 -show_entries stream=width,height -of json sample.ts
|
# ffprobe -loglevel quiet -select_streams v:0 -show_entries stream=width,height -of json sample.ts
|
||||||
@typechecked
|
@typechecked
|
||||||
def get_video_dimensions(ffprobe_path:str,
|
def get_video_dimensions(ffprobe_path:str, input_file: IO[bytes]) -> tuple[int,int]:
|
||||||
input_file: IO[bytes]) -> tuple[int,int]| tuple[None,None]:
|
|
||||||
infd = input_file.fileno()
|
infd = input_file.fileno()
|
||||||
lseek(infd, 0, SEEK_SET)
|
lseek(infd, 0, SEEK_SET)
|
||||||
set_inheritable(infd, True)
|
set_inheritable(infd, True)
|
||||||
@@ -188,8 +184,7 @@ def get_video_dimensions(ffprobe_path:str,
|
|||||||
if ('width' in video) and ('height' in video):
|
if ('width' in video) and ('height' in video):
|
||||||
return int(video['width']), int(video['height'])
|
return int(video['width']), int(video['height'])
|
||||||
|
|
||||||
logger.error('Impossible to retrieve dimensions of video')
|
raise InvalidMediaError('Impossible to retrieve dimensions of video')
|
||||||
return None, None
|
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def get_streams(ffprobe_path:str, input_file: IO[bytes]) -> list|None:
|
def get_streams(ffprobe_path:str, input_file: IO[bytes]) -> list|None:
|
||||||
@@ -244,7 +239,7 @@ def with_subtitles(ffprobe_path:str, input_file: IO[bytes]) -> bool:
|
|||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def find_subtitles_tracks(ffprobe_path:str, input_file: IO[bytes]) -> dict|None:
|
def find_subtitles_tracks(ffprobe_path:str, input_file: IO[bytes]) -> dict:
|
||||||
infd = input_file.fileno()
|
infd = input_file.fileno()
|
||||||
lseek(infd, 0, SEEK_SET)
|
lseek(infd, 0, SEEK_SET)
|
||||||
set_inheritable(infd, True)
|
set_inheritable(infd, True)
|
||||||
@@ -259,10 +254,9 @@ def find_subtitles_tracks(ffprobe_path:str, input_file: IO[bytes]) -> dict|None:
|
|||||||
out = json.load(BytesIO(out))
|
out = json.load(BytesIO(out))
|
||||||
if 'streams' in out:
|
if 'streams' in out:
|
||||||
return out['streams']
|
return out['streams']
|
||||||
logger.error('Impossible to retrieve format of file')
|
|
||||||
|
|
||||||
ffprobe.wait()
|
ffprobe.wait()
|
||||||
return None
|
raise InvalidMediaError('Impossible to retrieve format of file')
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def get_frames_in_stream(ffprobe_path: str, input_file: IO[bytes], begin:timedelta, end:timedelta,
|
def get_frames_in_stream(ffprobe_path: str, input_file: IO[bytes], begin:timedelta, end:timedelta,
|
||||||
@@ -280,8 +274,7 @@ def get_frames_in_stream(ffprobe_path: str, input_file: IO[bytes], begin:timedel
|
|||||||
frames = json.load(BytesIO(out))
|
frames = json.load(BytesIO(out))
|
||||||
status = ffprobe.wait()
|
status = ffprobe.wait()
|
||||||
if status != 0:
|
if status != 0:
|
||||||
logger.error('ffprobe failed with status code: %d', status)
|
raise ExternalToolError(f"ffprobe failed with status code: {status:d}")
|
||||||
return None
|
|
||||||
|
|
||||||
# Sort frames by timestamp
|
# Sort frames by timestamp
|
||||||
tmp = {}
|
tmp = {}
|
||||||
@@ -289,8 +282,6 @@ def get_frames_in_stream(ffprobe_path: str, input_file: IO[bytes], begin:timedel
|
|||||||
frames = frames['frames']
|
frames = frames['frames']
|
||||||
for frame in frames:
|
for frame in frames:
|
||||||
ts = get_ts_frame(frame)
|
ts = get_ts_frame(frame)
|
||||||
if ts is None:
|
|
||||||
return None
|
|
||||||
if begin <= ts <= end:
|
if begin <= ts <= end:
|
||||||
tmp[ts]=frame
|
tmp[ts]=frame
|
||||||
|
|
||||||
@@ -326,28 +317,25 @@ def get_nearest_idr_frame(ffprobe_path: str, input_file: IO[bytes], timestamp:ti
|
|||||||
frames = json.load(BytesIO(out))
|
frames = json.load(BytesIO(out))
|
||||||
status = ffprobe.wait()
|
status = ffprobe.wait()
|
||||||
if status != 0:
|
if status != 0:
|
||||||
logger.error('ffprobe failed with status code: %d', status)
|
raise ExternalToolError(f"ffprobe failed with status code: {status:d}")
|
||||||
return None
|
|
||||||
|
|
||||||
if 'frames' in frames:
|
if 'frames' in frames:
|
||||||
frames = frames['frames']
|
frames = frames['frames']
|
||||||
for frame in frames:
|
for frame in frames:
|
||||||
ts = get_ts_frame(frame)
|
ts = get_ts_frame(frame)
|
||||||
if ts is None:
|
|
||||||
return None
|
|
||||||
if tbegin <= ts <= tend:
|
if tbegin <= ts <= tend:
|
||||||
idrs.append(frame)
|
idrs.append(frame)
|
||||||
else:
|
else:
|
||||||
logger.error('Impossible to retrieve IDR frames inside file around [%s,%s]',
|
logger.error('Impossible to retrieve IDR frames inside file around [%s,%s]',
|
||||||
tbegin, tend)
|
tbegin, tend)
|
||||||
return None
|
return
|
||||||
|
|
||||||
return None
|
return
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def get_nearest_iframe(ffprobe_path:str, input_file: IO[bytes],
|
def get_nearest_iframe(ffprobe_path:str, input_file: IO[bytes],
|
||||||
timestamp:timedelta, before:bool=True,
|
timestamp:timedelta, before:bool=True,
|
||||||
delta_max:timedelta=timedelta(seconds=15))-> tuple[int,dict | None]:
|
delta_max:timedelta=timedelta(seconds=15))-> tuple[int,dict] | None:
|
||||||
infd = input_file.fileno()
|
infd = input_file.fileno()
|
||||||
set_inheritable(infd, True)
|
set_inheritable(infd, True)
|
||||||
|
|
||||||
@@ -356,14 +344,8 @@ def get_nearest_iframe(ffprobe_path:str, input_file: IO[bytes],
|
|||||||
iframe = None
|
iframe = None
|
||||||
while delta < delta_max:
|
while delta < delta_max:
|
||||||
zero = timedelta()
|
zero = timedelta()
|
||||||
if before:
|
tbegin = timestamp - delta if before else timestamp
|
||||||
tbegin = timestamp-delta
|
tend = timestamp + delta if not before else timestamp
|
||||||
else:
|
|
||||||
tbegin = timestamp
|
|
||||||
if not before:
|
|
||||||
tend = timestamp+delta
|
|
||||||
else:
|
|
||||||
tend = timestamp
|
|
||||||
tbegin = max(tbegin, zero)
|
tbegin = max(tbegin, zero)
|
||||||
logger.debug('Looking for an iframe in [%s, %s]', tbegin, tend)
|
logger.debug('Looking for an iframe in [%s, %s]', tbegin, tend)
|
||||||
|
|
||||||
@@ -381,8 +363,9 @@ def get_nearest_iframe(ffprobe_path:str, input_file: IO[bytes],
|
|||||||
|
|
||||||
found = False
|
found = False
|
||||||
for frame in iframes:
|
for frame in iframes:
|
||||||
|
try:
|
||||||
ts = get_ts_frame(frame)
|
ts = get_ts_frame(frame)
|
||||||
if ts is None:
|
except InvalidMediaError:
|
||||||
logger.warning('I-frame with no timestamp: %s', frame)
|
logger.warning('I-frame with no timestamp: %s', frame)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -404,11 +387,12 @@ def get_nearest_iframe(ffprobe_path:str, input_file: IO[bytes],
|
|||||||
its = get_ts_frame(iframe)
|
its = get_ts_frame(iframe)
|
||||||
if its is None:
|
if its is None:
|
||||||
logger.error("Impossible to retrieve timestamp of i-frame !")
|
logger.error("Impossible to retrieve timestamp of i-frame !")
|
||||||
return 0, None
|
return None
|
||||||
nb_frames = 0
|
nb_frames = 0
|
||||||
for frame in frames:
|
for frame in frames:
|
||||||
|
try:
|
||||||
ts = get_ts_frame(frame)
|
ts = get_ts_frame(frame)
|
||||||
if ts is None:
|
except InvalidMediaError:
|
||||||
logger.warning('Frame without timestamp: %s', frame)
|
logger.warning('Frame without timestamp: %s', frame)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -422,6 +406,6 @@ def get_nearest_iframe(ffprobe_path:str, input_file: IO[bytes],
|
|||||||
nb_frames = nb_frames+1
|
nb_frames = nb_frames+1
|
||||||
else:
|
else:
|
||||||
logger.error("Impossible to find I-frame between: %s and %s", tbegin, tend)
|
logger.error("Impossible to find I-frame between: %s and %s", tbegin, tend)
|
||||||
return 0, None
|
return None
|
||||||
|
|
||||||
return(nb_frames, iframe)
|
return(nb_frames, iframe)
|
||||||
|
|||||||
@@ -4,32 +4,27 @@
|
|||||||
|
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from io import TextIOWrapper
|
import os
|
||||||
import re
|
import re
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from os import (
|
from io import TextIOWrapper
|
||||||
read,
|
from os import SEEK_SET, fstat, ftruncate, lseek, read, set_inheritable, write
|
||||||
SEEK_SET,
|
from pathlib import Path
|
||||||
lseek,
|
|
||||||
write,
|
|
||||||
fstat,
|
|
||||||
set_inheritable,
|
|
||||||
ftruncate
|
|
||||||
)
|
|
||||||
import os
|
|
||||||
from subprocess import PIPE, Popen
|
from subprocess import PIPE, Popen
|
||||||
from typing import IO
|
from typing import IO, Sequence
|
||||||
|
|
||||||
from typeguard import typechecked
|
|
||||||
import hexdump
|
import hexdump
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
|
from typeguard import typechecked
|
||||||
|
|
||||||
|
from tscut.exceptions import InvalidMediaError, ExternalToolError
|
||||||
from tscut.matroska.ebml import change_ebml_element_size
|
from tscut.matroska.ebml import change_ebml_element_size
|
||||||
|
|
||||||
|
|
||||||
# Found codec private data using mkvinfo
|
# Found codec private data using mkvinfo
|
||||||
@typechecked
|
@typechecked
|
||||||
def get_codec_private_data_from_mkv(mkvinfo_path:str,
|
def get_codec_private_data_from_mkv(mkvinfo_path:str,
|
||||||
input_file: IO[bytes]) -> tuple[int, bytes]|tuple[None,None]:
|
input_file: IO[bytes]) -> tuple[int, bytes]:
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
infd = input_file.fileno()
|
infd = input_file.fileno()
|
||||||
@@ -64,8 +59,7 @@ def get_codec_private_data_from_mkv(mkvinfo_path:str,
|
|||||||
data = read(infd, size)
|
data = read(infd, size)
|
||||||
return position, data
|
return position, data
|
||||||
|
|
||||||
logger.error("Impossible to retrieve codec private data from MKV !")
|
raise InvalidMediaError("Impossible to retrieve codec private data from MKV !")
|
||||||
return None, None
|
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def parse_mkv_tree(mkvinfo_path:str, input_file: IO[bytes]) -> dict[str,tuple[int,int]]:
|
def parse_mkv_tree(mkvinfo_path:str, input_file: IO[bytes]) -> dict[str,tuple[int,int]]:
|
||||||
@@ -123,7 +117,8 @@ def parse_mkv_tree(mkvinfo_path:str, input_file: IO[bytes]) -> dict[str,tuple[in
|
|||||||
return elements
|
return elements
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def change_codec_private_data(mkvinfo_path:str, input_file: IO[bytes], codec_data:bytes) -> None:
|
def change_codec_private_data(mkvinfo_path:str, input_file: IO[bytes],
|
||||||
|
codec_data:bytes | bytearray) -> None:
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
infd = input_file.fileno()
|
infd = input_file.fileno()
|
||||||
@@ -133,7 +128,7 @@ def change_codec_private_data(mkvinfo_path:str, input_file: IO[bytes], codec_dat
|
|||||||
logger.info('Current size of file: %d', current_length)
|
logger.info('Current size of file: %d', current_length)
|
||||||
position, current_data = get_codec_private_data_from_mkv(mkvinfo_path, input_file)
|
position, current_data = get_codec_private_data_from_mkv(mkvinfo_path, input_file)
|
||||||
if position is None or current_data is None:
|
if position is None or current_data is None:
|
||||||
return None
|
raise InvalidMediaError("Impossible to retrieve private data from file")
|
||||||
current_data_length = len(current_data)
|
current_data_length = len(current_data)
|
||||||
future_length = current_length - current_data_length + len(codec_data)
|
future_length = current_length - current_data_length + len(codec_data)
|
||||||
logger.info('Expected size of file: %d', future_length)
|
logger.info('Expected size of file: %d', future_length)
|
||||||
@@ -151,8 +146,7 @@ def change_codec_private_data(mkvinfo_path:str, input_file: IO[bytes], codec_dat
|
|||||||
break
|
break
|
||||||
|
|
||||||
if not found:
|
if not found:
|
||||||
logger.error('Impossible to retrieve the key of codec private data')
|
raise InvalidMediaError("Impossible to retrieve the key of codec private data")
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
if current_length < future_length:
|
if current_length < future_length:
|
||||||
lseek(infd, position+current_data_length, SEEK_SET)
|
lseek(infd, position+current_data_length, SEEK_SET)
|
||||||
@@ -215,6 +209,7 @@ def extract_mkv_part(mkvmerge_path:str, input_file:IO[bytes], output_file:IO[byt
|
|||||||
logger.debug('Executing: %s', command)
|
logger.debug('Executing: %s', command)
|
||||||
|
|
||||||
with Popen(command, stdout=PIPE, close_fds=False, env=env) as mkvmerge:
|
with Popen(command, stdout=PIPE, close_fds=False, env=env) as mkvmerge:
|
||||||
|
assert mkvmerge.stdout is not None
|
||||||
pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%',
|
pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%',
|
||||||
desc='Extraction')
|
desc='Extraction')
|
||||||
for line in pb:
|
for line in pb:
|
||||||
@@ -223,6 +218,7 @@ def extract_mkv_part(mkvmerge_path:str, input_file:IO[bytes], output_file:IO[byt
|
|||||||
m = p.match(line)
|
m = p.match(line)
|
||||||
if m is None:
|
if m is None:
|
||||||
logger.error('Impossible to parse progress')
|
logger.error('Impossible to parse progress')
|
||||||
|
else:
|
||||||
pb.update(int(m['progress'])-pb.n)
|
pb.update(int(m['progress'])-pb.n)
|
||||||
elif line.startswith('Warning'):
|
elif line.startswith('Warning'):
|
||||||
warnings.append(line)
|
warnings.append(line)
|
||||||
@@ -236,14 +232,14 @@ def extract_mkv_part(mkvmerge_path:str, input_file:IO[bytes], output_file:IO[byt
|
|||||||
for w in warnings:
|
for w in warnings:
|
||||||
logger.warning(w)
|
logger.warning(w)
|
||||||
elif status == 2:
|
elif status == 2:
|
||||||
logger.error('Extraction returns errors')
|
raise ExternalToolError("Extraction returns errors")
|
||||||
|
|
||||||
|
|
||||||
# Merge a list of mkv files passed as input, and produce a new MKV as output
|
# Merge a list of mkv files passed as input, and produce a new MKV as output
|
||||||
@typechecked
|
@typechecked
|
||||||
def merge_mkvs(mkvmerge_path:str, inputs: list[IO[bytes]], output_name:str,
|
def merge_mkvs(mkvmerge_path:str, inputs: Sequence[IO[bytes]], output_name:str,
|
||||||
concatenate: bool=True,
|
concatenate: bool=True,
|
||||||
timestamps: dict[int, IO[str]] | None = None) -> IO[bytes]|None:
|
timestamps: dict[int, IO[str]] | None = None) -> IO[bytes]:
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
if timestamps is None:
|
if timestamps is None:
|
||||||
@@ -252,9 +248,9 @@ def merge_mkvs(mkvmerge_path:str, inputs: list[IO[bytes]], output_name:str,
|
|||||||
fds = []
|
fds = []
|
||||||
try:
|
try:
|
||||||
out = open(output_name, 'wb+')
|
out = open(output_name, 'wb+')
|
||||||
except OSError:
|
except OSError as e:
|
||||||
logger.error('Impossible to create file: %s', output_name)
|
logger.error('Impossible to create file: %s', output_name)
|
||||||
return None
|
raise e
|
||||||
|
|
||||||
outfd = out.fileno()
|
outfd = out.fileno()
|
||||||
lseek(outfd, 0, SEEK_SET)
|
lseek(outfd, 0, SEEK_SET)
|
||||||
@@ -297,6 +293,7 @@ def merge_mkvs(mkvmerge_path:str, inputs: list[IO[bytes]], output_name:str,
|
|||||||
logger.debug('Executing: LANG=C %s', merge_params)
|
logger.debug('Executing: LANG=C %s', merge_params)
|
||||||
|
|
||||||
with Popen(merge_params, stdout=PIPE, close_fds=False, env=env) as mkvmerge:
|
with Popen(merge_params, stdout=PIPE, close_fds=False, env=env) as mkvmerge:
|
||||||
|
assert mkvmerge.stdout is not None
|
||||||
pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%',
|
pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%',
|
||||||
desc='Merging')
|
desc='Merging')
|
||||||
for line in pb:
|
for line in pb:
|
||||||
@@ -305,6 +302,7 @@ def merge_mkvs(mkvmerge_path:str, inputs: list[IO[bytes]], output_name:str,
|
|||||||
m = p.match(line)
|
m = p.match(line)
|
||||||
if m is None:
|
if m is None:
|
||||||
logger.error('Impossible to parse progress')
|
logger.error('Impossible to parse progress')
|
||||||
|
else:
|
||||||
pb.n = int(m['progress'])
|
pb.n = int(m['progress'])
|
||||||
pb.update()
|
pb.update()
|
||||||
elif line.startswith('Warning'):
|
elif line.startswith('Warning'):
|
||||||
@@ -350,6 +348,7 @@ def extract_track_from_mkv(mkvextract_path: str, input_file: IO[bytes], index,
|
|||||||
logger.debug('Executing: LANG=C %s', params)
|
logger.debug('Executing: LANG=C %s', params)
|
||||||
|
|
||||||
with Popen(params, stdout=PIPE, close_fds=False, env=env) as extract:
|
with Popen(params, stdout=PIPE, close_fds=False, env=env) as extract:
|
||||||
|
assert extract.stdout is not None
|
||||||
pb = tqdm(TextIOWrapper(extract.stdout, encoding="utf-8"), total=100, unit='%',
|
pb = tqdm(TextIOWrapper(extract.stdout, encoding="utf-8"), total=100, unit='%',
|
||||||
desc='Extraction of track')
|
desc='Extraction of track')
|
||||||
for line in pb:
|
for line in pb:
|
||||||
@@ -358,6 +357,7 @@ def extract_track_from_mkv(mkvextract_path: str, input_file: IO[bytes], index,
|
|||||||
m = p.match(line)
|
m = p.match(line)
|
||||||
if m is None:
|
if m is None:
|
||||||
logger.error('Impossible to parse progress')
|
logger.error('Impossible to parse progress')
|
||||||
|
else:
|
||||||
pb.update(int(m['progress'])-pb.n)
|
pb.update(int(m['progress'])-pb.n)
|
||||||
pb.update(100-pb.n)
|
pb.update(100-pb.n)
|
||||||
pb.refresh()
|
pb.refresh()
|
||||||
@@ -366,7 +366,7 @@ def extract_track_from_mkv(mkvextract_path: str, input_file: IO[bytes], index,
|
|||||||
extract.wait()
|
extract.wait()
|
||||||
|
|
||||||
if extract.returncode != 0:
|
if extract.returncode != 0:
|
||||||
logger.error('Mkvextract returns an error code: %d', extract.returncode)
|
raise ExternalToolError(f"Mkvextract returns an error code: {extract.returncode:d}")
|
||||||
else:
|
else:
|
||||||
logger.info('Track %d was succesfully extracted.', index)
|
logger.info('Track %d was succesfully extracted.', index)
|
||||||
|
|
||||||
@@ -389,6 +389,7 @@ def remove_video_tracks_from_mkv(mkvmerge_path:str, input_file: IO[bytes],
|
|||||||
|
|
||||||
env = {**os.environ, 'LANG': 'C'}
|
env = {**os.environ, 'LANG': 'C'}
|
||||||
with Popen(params, stdout=PIPE, close_fds=False, env=env) as remove:
|
with Popen(params, stdout=PIPE, close_fds=False, env=env) as remove:
|
||||||
|
assert remove.stdout is not None
|
||||||
pb = tqdm(TextIOWrapper(remove.stdout, encoding="utf-8"), total=100, unit='%',
|
pb = tqdm(TextIOWrapper(remove.stdout, encoding="utf-8"), total=100, unit='%',
|
||||||
desc='Removal of video track:')
|
desc='Removal of video track:')
|
||||||
for line in pb:
|
for line in pb:
|
||||||
@@ -397,6 +398,7 @@ def remove_video_tracks_from_mkv(mkvmerge_path:str, input_file: IO[bytes],
|
|||||||
m = p.match(line)
|
m = p.match(line)
|
||||||
if m is None:
|
if m is None:
|
||||||
logger.error('Impossible to parse progress')
|
logger.error('Impossible to parse progress')
|
||||||
|
else:
|
||||||
pb.update(int(m['progress'])-pb.n)
|
pb.update(int(m['progress'])-pb.n)
|
||||||
pb.update(100-pb.n)
|
pb.update(100-pb.n)
|
||||||
pb.refresh()
|
pb.refresh()
|
||||||
@@ -405,20 +407,20 @@ def remove_video_tracks_from_mkv(mkvmerge_path:str, input_file: IO[bytes],
|
|||||||
remove.wait()
|
remove.wait()
|
||||||
|
|
||||||
if remove.returncode != 0:
|
if remove.returncode != 0:
|
||||||
logger.error('Mkvmerge returns an error code: %d', remove.returncode)
|
raise ExternalToolError(f"Mkvmerge returns an error code: {remove.returncode:d}")
|
||||||
else:
|
else:
|
||||||
logger.info('Video tracks were succesfully extracted.')
|
logger.info('Video tracks were succesfully extracted.')
|
||||||
|
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def remux_srt_subtitles(mkvmerge_path:str, input_file: IO[bytes], output_filename: str,
|
def remux_srt_subtitles(mkvmerge_path:str, input_file: IO[bytes], output_file: Path,
|
||||||
subtitles) -> None:
|
subtitles) -> None:
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
out = open(output_filename, 'w', encoding='utf8')
|
out = open(output_file, 'w', encoding='utf8')
|
||||||
except OSError:
|
except OSError:
|
||||||
logger.error('Impossible to create file: %s', output_filename)
|
logger.error('Impossible to create file: %s', output_file)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
outfd = out.fileno()
|
outfd = out.fileno()
|
||||||
@@ -439,6 +441,7 @@ def remux_srt_subtitles(mkvmerge_path:str, input_file: IO[bytes], output_filenam
|
|||||||
env = {**os.environ, 'LANG': 'C'}
|
env = {**os.environ, 'LANG': 'C'}
|
||||||
logger.info('Remux subtitles: %s', mkv_merge_params)
|
logger.info('Remux subtitles: %s', mkv_merge_params)
|
||||||
with Popen(mkv_merge_params, stdout=PIPE, close_fds=False, env=env) as mkvmerge:
|
with Popen(mkv_merge_params, stdout=PIPE, close_fds=False, env=env) as mkvmerge:
|
||||||
|
assert mkvmerge.stdout is not None
|
||||||
pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%',
|
pb = tqdm(TextIOWrapper(mkvmerge.stdout, encoding="utf-8"), total=100, unit='%',
|
||||||
desc='Remux subtitles:')
|
desc='Remux subtitles:')
|
||||||
for line in pb:
|
for line in pb:
|
||||||
@@ -447,6 +450,7 @@ def remux_srt_subtitles(mkvmerge_path:str, input_file: IO[bytes], output_filenam
|
|||||||
m = p.match(line)
|
m = p.match(line)
|
||||||
if m is None:
|
if m is None:
|
||||||
logger.error('Impossible to parse progress')
|
logger.error('Impossible to parse progress')
|
||||||
|
else:
|
||||||
pb.n = int(m['progress'])
|
pb.n = int(m['progress'])
|
||||||
pb.update()
|
pb.update()
|
||||||
elif line.startswith('Warning'):
|
elif line.startswith('Warning'):
|
||||||
|
|||||||
@@ -5,16 +5,19 @@
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
from io import BytesIO
|
||||||
from math import ceil, log
|
from math import ceil, log
|
||||||
from os import write
|
from os import write
|
||||||
from typing import IO
|
from typing import IO
|
||||||
from io import BytesIO
|
|
||||||
|
|
||||||
|
|
||||||
from typeguard import typechecked
|
from typeguard import typechecked
|
||||||
|
|
||||||
|
from tscut.exceptions import TemporaryFileError
|
||||||
|
from tscut.temporaries import TemporaryFiles
|
||||||
|
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def dump_ppm(pictures: bytes, prefix: str, temporaries: list[IO[bytes]]) -> None:
|
def dump_ppm(pictures: bytes, prefix: str, temporaries: TemporaryFiles) -> None:
|
||||||
"""
|
"""
|
||||||
Dump PPM pictures from a bytes buffer to files.
|
Dump PPM pictures from a bytes buffer to files.
|
||||||
|
|
||||||
@@ -71,7 +74,7 @@ def dump_ppm(pictures: bytes, prefix: str, temporaries: list[IO[bytes]]) -> None
|
|||||||
header_len=2+1+ceil(log(width, 10))+1+ceil(log(height, 10))+1+3+1
|
header_len=2+1+ceil(log(width, 10))+1+ceil(log(height, 10))+1+3+1
|
||||||
try:
|
try:
|
||||||
with open(filename, 'wb') as out:
|
with open(filename, 'wb') as out:
|
||||||
temporaries.append(out)
|
temporaries.add(out)
|
||||||
outfd = out.fileno()
|
outfd = out.fileno()
|
||||||
length=header_len+3*width*height
|
length=header_len+3*width*height
|
||||||
nb_bytes = 0
|
nb_bytes = 0
|
||||||
@@ -79,5 +82,5 @@ def dump_ppm(pictures: bytes, prefix: str, temporaries: list[IO[bytes]]) -> None
|
|||||||
nb_bytes+=write(outfd, pictures[pos+nb_bytes:pos+length])
|
nb_bytes+=write(outfd, pictures[pos+nb_bytes:pos+length])
|
||||||
pos+=length
|
pos+=length
|
||||||
picture+=1
|
picture+=1
|
||||||
except OSError:
|
except OSError as e:
|
||||||
logger.error('Impossible to create file: %s', filename)
|
raise TemporaryFileError(f"Impossible to create file {filename}") from e
|
||||||
|
|||||||
@@ -9,6 +9,9 @@ from datetime import timedelta
|
|||||||
|
|
||||||
from typeguard import typechecked
|
from typeguard import typechecked
|
||||||
|
|
||||||
|
from tscut.exceptions import InvalidMediaError
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def parse_timestamp(ts:str) -> timedelta|None:
|
def parse_timestamp(ts:str) -> timedelta|None:
|
||||||
@@ -29,8 +32,6 @@ def parse_timestamp(ts:str) -> timedelta|None:
|
|||||||
- the timestamp string is not in the correct format
|
- the timestamp string is not in the correct format
|
||||||
- the timestamp values are out of range (e.g. hour > 23, minute > 59, etc.)
|
- the timestamp values are out of range (e.g. hour > 23, minute > 59, etc.)
|
||||||
"""
|
"""
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
ts_reg_exp = (r'^(?P<hour>[0-9]{1,2}):(?P<minute>[0-9]{1,2})'
|
ts_reg_exp = (r'^(?P<hour>[0-9]{1,2}):(?P<minute>[0-9]{1,2})'
|
||||||
r':(?P<second>[0-9]{1,2})(\.(?P<us>[0-9]{1,6}))?$')
|
r':(?P<second>[0-9]{1,2})(\.(?P<us>[0-9]{1,6}))?$')
|
||||||
p = re.compile(ts_reg_exp)
|
p = re.compile(ts_reg_exp)
|
||||||
@@ -65,9 +66,8 @@ def parse_timestamp(ts:str) -> timedelta|None:
|
|||||||
if us < 0 or us > 1000000:
|
if us < 0 or us > 1000000:
|
||||||
logger.error("milliseconds must be in [0,1000000[")
|
logger.error("milliseconds must be in [0,1000000[")
|
||||||
return None
|
return None
|
||||||
res = timedelta(hours=hour, minutes=minute, seconds=second, microseconds=us)
|
|
||||||
|
|
||||||
return res
|
return timedelta(hours=hour, minutes=minute, seconds=second, microseconds=us)
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def parse_time_interval(interval: str) -> tuple[timedelta, timedelta] | tuple[None, None]:
|
def parse_time_interval(interval: str) -> tuple[timedelta, timedelta] | tuple[None, None]:
|
||||||
@@ -90,8 +90,6 @@ def parse_time_interval(interval: str) -> tuple[timedelta, timedelta] | tuple[No
|
|||||||
- the time values are out of range (e.g. hour > 23, minute > 59, etc.)
|
- the time values are out of range (e.g. hour > 23, minute > 59, etc.)
|
||||||
- the end time is before the start time (non-monotonic interval)
|
- the end time is before the start time (non-monotonic interval)
|
||||||
"""
|
"""
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
interval_reg_exp = (r'^(?P<hour1>[0-9]{1,2}):(?P<minute1>[0-9]{1,2}):(?P<second1>[0-9]{1,2})'
|
interval_reg_exp = (r'^(?P<hour1>[0-9]{1,2}):(?P<minute1>[0-9]{1,2}):(?P<second1>[0-9]{1,2})'
|
||||||
r'(\.(?P<ms1>[0-9]{1,3}))?-(?P<hour2>[0-9]{1,2}):(?P<minute2>[0-9]{1,2})'
|
r'(\.(?P<ms1>[0-9]{1,3}))?-(?P<hour2>[0-9]{1,2}):(?P<minute2>[0-9]{1,2})'
|
||||||
r':(?P<second2>[0-9]{1,2})(\.(?P<ms2>[0-9]{1,3}))?$')
|
r':(?P<second2>[0-9]{1,2})(\.(?P<ms2>[0-9]{1,3}))?$')
|
||||||
@@ -193,30 +191,23 @@ def compare_time_interval(interval1: tuple[timedelta, timedelta],
|
|||||||
|
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def get_ts_frame(frame: dict) -> timedelta|None:
|
def get_ts_frame(frame: dict) -> timedelta:
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
if 'pts_time' in frame:
|
if 'pts_time' in frame:
|
||||||
pts_time = float(frame['pts_time'])
|
pts_time = float(frame['pts_time'])
|
||||||
elif 'pkt_pts_time' in frame:
|
elif 'pkt_pts_time' in frame:
|
||||||
pts_time = float(frame['pkt_pts_time'])
|
pts_time = float(frame['pkt_pts_time'])
|
||||||
else:
|
else:
|
||||||
logger.error('Impossible to find timestamp of frame %s', frame)
|
raise InvalidMediaError(f"Impossible to find timestamp of frame {frame}")
|
||||||
return None
|
|
||||||
|
|
||||||
ts = timedelta(seconds=pts_time)
|
return timedelta(seconds=pts_time)
|
||||||
return ts
|
|
||||||
|
|
||||||
@typechecked
|
@typechecked
|
||||||
def get_packet_duration(packet: dict) -> int | None:
|
def get_packet_duration(packet: dict) -> int:
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
if 'duration' in packet:
|
if 'duration' in packet:
|
||||||
duration = int(packet['duration'])
|
duration = int(packet['duration'])
|
||||||
elif 'pkt_duration' in packet:
|
elif 'pkt_duration' in packet:
|
||||||
duration = int(packet['pkt_duration'])
|
duration = int(packet['pkt_duration'])
|
||||||
else:
|
else:
|
||||||
logger.error('Impossible to find duration of packet %s', packet)
|
raise InvalidMediaError("Impossible to find duration of packet {packet}")
|
||||||
return None
|
|
||||||
|
|
||||||
return duration
|
return duration
|
||||||
|
|||||||
@@ -1,193 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
'''A module to remove parts of video (.e.g advertisements) with single frame precision.'''
|
|
||||||
|
|
||||||
# Standard modules
|
|
||||||
import logging
|
|
||||||
from enum import IntEnum, unique
|
|
||||||
from os import (
|
|
||||||
SEEK_SET,
|
|
||||||
fstat,
|
|
||||||
lseek,
|
|
||||||
read,
|
|
||||||
write,
|
|
||||||
)
|
|
||||||
from sys import exit
|
|
||||||
from typing import IO
|
|
||||||
|
|
||||||
# Third party libraries
|
|
||||||
from tqdm import tqdm
|
|
||||||
from typeguard import typechecked
|
|
||||||
|
|
||||||
from tscut.tools.mkvtoolnix import change_codec_private_data
|
|
||||||
|
|
||||||
from tscut.h264.avc import (
|
|
||||||
get_avc_config_from_h264,
|
|
||||||
parse_codec_private
|
|
||||||
)
|
|
||||||
from tscut.matroska.ebml import change_ebml_element_size
|
|
||||||
from tscut.matroska.codec import dump_codec_private_data
|
|
||||||
from tscut.tools.ffprobe import (
|
|
||||||
get_format,
|
|
||||||
get_frame_rate,
|
|
||||||
get_video_dimensions,
|
|
||||||
get_movie_duration,
|
|
||||||
get_streams,
|
|
||||||
with_subtitles,
|
|
||||||
find_subtitles_tracks,
|
|
||||||
get_nearest_iframe,
|
|
||||||
get_ts_frame
|
|
||||||
)
|
|
||||||
from tscut.tools.ffmpeg import (
|
|
||||||
extract_all_streams,
|
|
||||||
ffmpeg_convert
|
|
||||||
)
|
|
||||||
from tscut.tools.timeframe import (
|
|
||||||
compare_time_interval,
|
|
||||||
parse_time_interval
|
|
||||||
)
|
|
||||||
from tscut.tools.discovery import check_required_tools
|
|
||||||
from tscut.ocr.subtitles import (
|
|
||||||
do_ocr,
|
|
||||||
extract_srt,get_tesseract_supported_lang
|
|
||||||
)
|
|
||||||
from tscut.tools.mkvtoolnix import (
|
|
||||||
extract_mkv_part,
|
|
||||||
extract_track_from_mkv,
|
|
||||||
get_codec_private_data_from_mkv,
|
|
||||||
merge_mkvs,
|
|
||||||
remove_video_tracks_from_mkv,
|
|
||||||
remux_srt_subtitles
|
|
||||||
)
|
|
||||||
|
|
||||||
# Useful SPS/PPS discussion.
|
|
||||||
# https://copyprogramming.com/howto/including-sps-and-pps-in-a-raw-h264-track
|
|
||||||
# https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390
|
|
||||||
|
|
||||||
# New strategy: a possible way of handling multiple SPS/PPS gracefully.
|
|
||||||
# Encode each head and trailer with FFMPEG using only I-frame (to be sure the NAL unit will never
|
|
||||||
# refer to another image).
|
|
||||||
# Encode using an different SPS-ID all of them (using sps-id parameter of libx264 library, e.g
|
|
||||||
# 1 instead of 0).
|
|
||||||
# For the video track produce only a raw H264 file and a file containing timestamps of the
|
|
||||||
# different frames.
|
|
||||||
# For the rest of the tracks (audio, subtitles) produce directly a MKV (this is already done).
|
|
||||||
# Concatenate all raw H264 in a giant one (like cat), and the same for timestamps of video frames
|
|
||||||
# (to keep sound and video synchronized).
|
|
||||||
# Then use mkvmerge to remux the H264 track and the rest of tracks.
|
|
||||||
# MKVmerge "concatenate" subcommand is able to concatenate different SPS/PPS data into a bigger
|
|
||||||
# Private Codec Data.
|
|
||||||
# However, this is proved to be not reliable. Sometimes it results in a AVC context containing
|
|
||||||
# a single SPS/PPS.
|
|
||||||
# So we have to rely on a manual parsing of the H264 AVC context of original movie
|
|
||||||
# and the ones produced for headers and trailers, and then merging them into a bigger AVC context.
|
|
||||||
# Then finally, change the Private Codec Data in the final MKV.
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@unique
|
|
||||||
class SupportedFormat(IntEnum):
|
|
||||||
TS = 1
|
|
||||||
MP4 = 2
|
|
||||||
MATROSKA = 3
|
|
||||||
|
|
||||||
def __str__(self):
|
|
||||||
match self:
|
|
||||||
case SupportedFormat.TS:
|
|
||||||
return 'mpegts'
|
|
||||||
case SupportedFormat.MP4:
|
|
||||||
return 'mov,mp4,m4a,3gp,3g2,mj2'
|
|
||||||
case SupportedFormat.MATROSKA:
|
|
||||||
return 'matroska,webm'
|
|
||||||
case _:
|
|
||||||
return 'Unsupported format'
|
|
||||||
|
|
||||||
# Extract SPS/PPS
|
|
||||||
# https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390
|
|
||||||
# ffmpeg -i <InputFile (before concatenation)> -c:v copy -an -sn -bsf:v trace_headers -t 0.01\
|
|
||||||
# -report -loglevel 0 -f null -
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@typechecked
|
|
||||||
def concatenate_h264_parts(h264parts: list[IO[bytes]], output: IO[bytes]) -> None:
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
total_length = 0
|
|
||||||
for h264 in h264parts:
|
|
||||||
fd = h264.fileno()
|
|
||||||
total_length += fstat(fd).st_size
|
|
||||||
|
|
||||||
logger.info('Total length: %d', total_length)
|
|
||||||
|
|
||||||
outfd = output.fileno()
|
|
||||||
lseek(outfd, 0, SEEK_SET)
|
|
||||||
|
|
||||||
pb = tqdm(total=total_length, unit='bytes', desc='Concatenation')
|
|
||||||
for h264 in h264parts:
|
|
||||||
fd = h264.fileno()
|
|
||||||
lseek(fd, 0, SEEK_SET)
|
|
||||||
while True:
|
|
||||||
buf = read(fd, 1000000)
|
|
||||||
if buf is None or len(buf) == 0:
|
|
||||||
break
|
|
||||||
pos = 0
|
|
||||||
while pos < len(buf):
|
|
||||||
nb_bytes = write(outfd, buf[pos:])
|
|
||||||
pb.update(nb_bytes)
|
|
||||||
pos += nb_bytes
|
|
||||||
|
|
||||||
def concatenate_h264_ts_parts(h264_ts_parts: list[IO[bytes]], output: IO[bytes]) -> None:
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
header = '# timestamp format v2\n'.encode('ascii')
|
|
||||||
|
|
||||||
output.write(header)
|
|
||||||
|
|
||||||
last = 0.
|
|
||||||
first = True
|
|
||||||
for part in h264_ts_parts:
|
|
||||||
if first:
|
|
||||||
offset = last
|
|
||||||
else:
|
|
||||||
# TODO: take framerate into account
|
|
||||||
offset = last + 40
|
|
||||||
logger.debug('Parsing file: %s. Offset=%d', part, offset)
|
|
||||||
isheader = part.readline()
|
|
||||||
if (not isheader) or (isheader != header):
|
|
||||||
logger.error('Impossible to find a valid header: "%s"', isheader)
|
|
||||||
exit(-1)
|
|
||||||
while True:
|
|
||||||
line = part.readline()
|
|
||||||
if not line:
|
|
||||||
break
|
|
||||||
ts = offset + float(line)
|
|
||||||
last = max(last,ts)
|
|
||||||
output.write(f'{ts:f}\n'.encode('ascii'))
|
|
||||||
if first:
|
|
||||||
first = False
|
|
||||||
|
|
||||||
# TODO: finish this procedure
|
|
||||||
def do_coarse_processing(ffmpeg_path:str, ffprobe_path:str, mkvmerge_path:str,
|
|
||||||
input_file: IO[bytes], begin, end, nb_frames, framerate,
|
|
||||||
files_prefix, streams, width, height, temporaries, dump_mem_fd) -> None:
|
|
||||||
# pylint: disable=W0613
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
# Internal video with all streams (video, audio and subtitles)
|
|
||||||
internal_mkv_name = f'{files_prefix}.mkv'
|
|
||||||
|
|
||||||
try:
|
|
||||||
internal_mkv = open(internal_mkv_name, 'wb+')
|
|
||||||
except OSError:
|
|
||||||
logger.error('Impossible to create file: %s', internal_mkv_name)
|
|
||||||
exit(-1)
|
|
||||||
|
|
||||||
# Extract internal part of MKV
|
|
||||||
extract_mkv_part(mkvmerge_path=mkvmerge_path, input_file=input_file, output_file=internal_mkv,
|
|
||||||
begin=begin, end=end)
|
|
||||||
|
|
||||||
temporaries.append(internal_mkv)
|
|
||||||
Reference in New Issue
Block a user