From d756fede54ee14d89f2bbc59f2bac32426dfe067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Tronel?= Date: Sun, 30 Aug 2026 17:45:03 +0200 Subject: [PATCH] H264 concatenation functions are moved to cutting module. --- src/tscut/cli.py | 2 +- src/tscut/coarse.py | 6 --- src/tscut/cutting.py | 68 ++++++++++++++++++++++- src/tscut/docs/README.md | 26 +++++++++ src/tscut/tscut.py | 113 --------------------------------------- 5 files changed, 93 insertions(+), 122 deletions(-) create mode 100644 src/tscut/docs/README.md delete mode 100755 src/tscut/tscut.py diff --git a/src/tscut/cli.py b/src/tscut/cli.py index d67c104..4e0bc35 100644 --- a/src/tscut/cli.py +++ b/src/tscut/cli.py @@ -12,8 +12,8 @@ from pathlib import Path import coloredlogs -from tscut.models import ProcessingOptions from tscut.exceptions import TSCutError +from tscut.models import ProcessingOptions from tscut.pipeline import process_recording from tscut.tools.discovery import check_required_tools from tscut.tools.timeframe import compare_time_interval, parse_time_interval diff --git a/src/tscut/coarse.py b/src/tscut/coarse.py index 0e280a5..c10a235 100644 --- a/src/tscut/coarse.py +++ b/src/tscut/coarse.py @@ -12,16 +12,10 @@ from tscut.tools.mkvtoolnix import ( logger = logging.getLogger(__name__) - - - # TODO: finish this procedure def do_coarse_processing(ffmpeg_path:str, ffprobe_path:str, mkvmerge_path:str, input_file: IO[bytes], begin, end, nb_frames, framerate, files_prefix, streams, width, height, temporaries, dump_mem_fd) -> None: - # pylint: disable=W0613 - logger = logging.getLogger(__name__) - # Internal video with all streams (video, audio and subtitles) internal_mkv_name = f'{files_prefix}.mkv' diff --git a/src/tscut/cutting.py b/src/tscut/cutting.py index 677dd28..f97875b 100644 --- a/src/tscut/cutting.py +++ b/src/tscut/cutting.py @@ -4,10 +4,19 @@ import logging from datetime import timedelta +from os import ( + SEEK_SET, + fstat, + lseek, + read, + write, +) from shutil import copyfile -from typing import IO, Any +from typing import IO, Any, BinaryIO, Sequence, TextIO import hexdump +from tqdm import tqdm +from typeguard import typechecked from tscut.exceptions import InvalidMediaError from tscut.h264.avc import get_avc_config_from_h264 @@ -25,11 +34,66 @@ from tscut.tools.mkvtoolnix import ( remove_video_tracks_from_mkv, ) from tscut.tools.timeframe import get_ts_frame -from tscut.tscut import concatenate_h264_parts, concatenate_h264_ts_parts logger = logging.getLogger(__name__) +@typechecked +def concatenate_h264_parts(h264parts: Sequence[BinaryIO], output: BinaryIO) -> None: + logger = logging.getLogger(__name__) + + total_length = 0 + for h264 in h264parts: + fd = h264.fileno() + total_length += fstat(fd).st_size + + logger.info('Total length: %d', total_length) + + outfd = output.fileno() + lseek(outfd, 0, SEEK_SET) + + pb = tqdm(total=total_length, unit='bytes', desc='Concatenation') + for h264 in h264parts: + fd = h264.fileno() + lseek(fd, 0, SEEK_SET) + while True: + buf = read(fd, 1000000) + if buf is None or len(buf) == 0: + break + pos = 0 + while pos < len(buf): + nb_bytes = write(outfd, buf[pos:]) + pb.update(nb_bytes) + pos += nb_bytes + +def concatenate_h264_ts_parts(h264_ts_parts: Sequence[TextIO], output: TextIO) -> None: + logger = logging.getLogger(__name__) + header = '# timestamp format v2\n' + output.write(header) + + last = 0. + first = True + for part in h264_ts_parts: + if first: + offset = last + else: + # TODO: take framerate into account + offset = last + 40 + logger.debug('Parsing file: %s. Offset=%d', part, offset) + isheader = part.readline() + if (not isheader) or (isheader != header): + logger.error('Impossible to find a valid header: "%s"', isheader) + exit(-1) + while True: + line = part.readline() + if not line: + break + ts = offset + float(line) + last = max(last,ts) + output.write(f'{ts:f}\n') + if first: + first = False + def cut_recording(media: PreparedMedia, options: ProcessingOptions, temporaries: list[IO[Any]]) -> CutResult: diff --git a/src/tscut/docs/README.md b/src/tscut/docs/README.md new file mode 100644 index 0000000..a528c65 --- /dev/null +++ b/src/tscut/docs/README.md @@ -0,0 +1,26 @@ + +Useful SPS/PPS discussion: +1.https://copyprogramming.com/howto/including-sps-and-pps-in-a-raw-h264-track +2. https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390 + +Strategy: a possible way of handling multiple SPS/PPS gracefully. +Encode each head and trailer with FFMPEG using only I-frame (to be sure the NAL unit will never refer to another image). +Encode using an different SPS-ID all of them (using sps-id parameter of libx264 library, e.g 1 instead of 0). +For the video track produce only a raw H264 file and a file containing timestamps of the different frames. +For the rest of the tracks (audio, subtitles) produce directly a MKV (this is already done). +Concatenate all raw H264 in a giant one (like cat), and the same for timestamps of video frames (to keep sound and video synchronized). +Then use mkvmerge to remux the H264 track and the rest of tracks. +MKVmerge "concatenate" subcommand is able to concatenate different SPS/PPS data into a bigger Private Codec Data. +However, this is proved to be not reliable. +Sometimes it results in a AVC context containing a single SPS/PPS. +So we have to rely on a manual parsing of the H264 AVC context of original movie and the ones produced for headers and trailers, and then merging them into a bigger AVC context. +Then finally, change the Private Codec Data in the final MKV. + + +Extract SPS/PPS: +1. https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390 +```bash +ffmpeg -i -c:v copy -an -sn -bsf:v trace_headers -t 0.01 -report -loglevel 0 -f null - +`` + + diff --git a/src/tscut/tscut.py b/src/tscut/tscut.py deleted file mode 100755 index 7034794..0000000 --- a/src/tscut/tscut.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 -'''A module to remove parts of video (.e.g advertisements) with single frame precision.''' - -# Standard modules -import logging -from os import ( - SEEK_SET, - fstat, - lseek, - read, - write, -) -from sys import exit -from typing import IO, BinaryIO, Sequence, TextIO - -# Third party libraries -from tqdm import tqdm -from typeguard import typechecked - -from tscut.models import SupportedFormat -from tscut.tools.mkvtoolnix import ( - extract_mkv_part, -) - -# Useful SPS/PPS discussion. -# https://copyprogramming.com/howto/including-sps-and-pps-in-a-raw-h264-track -# https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390 - -# New strategy: a possible way of handling multiple SPS/PPS gracefully. -# Encode each head and trailer with FFMPEG using only I-frame (to be sure the NAL unit will never -# refer to another image). -# Encode using an different SPS-ID all of them (using sps-id parameter of libx264 library, e.g -# 1 instead of 0). -# For the video track produce only a raw H264 file and a file containing timestamps of the -# different frames. -# For the rest of the tracks (audio, subtitles) produce directly a MKV (this is already done). -# Concatenate all raw H264 in a giant one (like cat), and the same for timestamps of video frames -# (to keep sound and video synchronized). -# Then use mkvmerge to remux the H264 track and the rest of tracks. -# MKVmerge "concatenate" subcommand is able to concatenate different SPS/PPS data into a bigger -# Private Codec Data. -# However, this is proved to be not reliable. Sometimes it results in a AVC context containing -# a single SPS/PPS. -# So we have to rely on a manual parsing of the H264 AVC context of original movie -# and the ones produced for headers and trailers, and then merging them into a bigger AVC context. -# Then finally, change the Private Codec Data in the final MKV. - - -# Extract SPS/PPS -# https://gitlab.com/mbunkus/mkvtoolnix/-/issues/2390 -# ffmpeg -i -c:v copy -an -sn -bsf:v trace_headers -t 0.01\ -# -report -loglevel 0 -f null - - - - - -@typechecked -def concatenate_h264_parts(h264parts: Sequence[BinaryIO], output: BinaryIO) -> None: - logger = logging.getLogger(__name__) - - total_length = 0 - for h264 in h264parts: - fd = h264.fileno() - total_length += fstat(fd).st_size - - logger.info('Total length: %d', total_length) - - outfd = output.fileno() - lseek(outfd, 0, SEEK_SET) - - pb = tqdm(total=total_length, unit='bytes', desc='Concatenation') - for h264 in h264parts: - fd = h264.fileno() - lseek(fd, 0, SEEK_SET) - while True: - buf = read(fd, 1000000) - if buf is None or len(buf) == 0: - break - pos = 0 - while pos < len(buf): - nb_bytes = write(outfd, buf[pos:]) - pb.update(nb_bytes) - pos += nb_bytes - -def concatenate_h264_ts_parts(h264_ts_parts: Sequence[TextIO], output: TextIO) -> None: - logger = logging.getLogger(__name__) - header = '# timestamp format v2\n' - output.write(header) - - last = 0. - first = True - for part in h264_ts_parts: - if first: - offset = last - else: - # TODO: take framerate into account - offset = last + 40 - logger.debug('Parsing file: %s. Offset=%d', part, offset) - isheader = part.readline() - if (not isheader) or (isheader != header): - logger.error('Impossible to find a valid header: "%s"', isheader) - exit(-1) - while True: - line = part.readline() - if not line: - break - ts = offset + float(line) - last = max(last,ts) - output.write(f'{ts:f}\n') - if first: - first = False - -