From 81912155c184816c922a58b999fbc9bd6214fb6e Mon Sep 17 00:00:00 2001 From: Leo Butler Date: Sun, 28 Jun 2020 17:30:57 -0500 Subject: [PATCH] core sh scripts for transcoding to h264 in mkv --- ffmpeg-cut-to-mkv.sh | 18 ++++++ ffmpeg-to-mkv-marks.awk | 38 +++++++++++++ ffmpeg-to-mkv-seektable.awk | 33 +++++++++++ ffmpeg-to-mkv.sh | 55 ++++++++++++++++++ myth-rebuild-seektable.sh | 109 ++++++++++++++++++++++++++++++++++++ myth-update-basename.sh | 12 ++++ 6 files changed, 265 insertions(+) create mode 100755 ffmpeg-cut-to-mkv.sh create mode 100644 ffmpeg-to-mkv-marks.awk create mode 100644 ffmpeg-to-mkv-seektable.awk create mode 100755 ffmpeg-to-mkv.sh create mode 100755 myth-rebuild-seektable.sh create mode 100755 myth-update-basename.sh diff --git a/ffmpeg-cut-to-mkv.sh b/ffmpeg-cut-to-mkv.sh new file mode 100755 index 0000000..fc7fa5e --- /dev/null +++ b/ffmpeg-cut-to-mkv.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -a +set -e +set -x + +file=${file:-"$A1"} +recordingsdir=${A2:-"/var/lib/mythtv/recordings"} +chanid=$1 +starttimeutc=$2 +title=${title:-"$A3"} +subtitle=${subtitle:-"$A4"} +season=${season:-"$A5"} +episode=${episode:-"$A6"} + +ffmpeg-cut-list0.sh $chanid $starttimeutc +ffmpeg-to-mkv.sh $chanid $starttimeutc + +exit $? diff --git a/ffmpeg-to-mkv-marks.awk b/ffmpeg-to-mkv-marks.awk new file mode 100644 index 0000000..3307e0f --- /dev/null +++ b/ffmpeg-to-mkv-marks.awk @@ -0,0 +1,38 @@ +BEGIN { + RS="\n" + ORS="\n" + FS="[=/]" + mark["width"]=30 + mark["height"]=31 + mark["avg_frame_rate"]=32 + mark["duration"]=33 + mark["total_frames"]=34 +} +# A typical record looks like: (I don't know how to remove duplicates) +# width=1280 +# height=720 +# avg_frame_rate=60000/1001 +# duration=719.385322 +# width=1280 +# height=720 +# avg_frame_rate=60000/1001 +# duration=719.385322 +# -- +/width|height|avg_frame_rate|duration/{ + if($1 ~ /avg_frame_rate|duration/) { + if (NF == 3 && $2 ~ /[0-9]+/ && $3 ~ /[1-9][0-9.]*/) { + #print $2 " " $3 + a[$1] = int($2/$3*1000+0.5) + } else { + a[$1] = int($2*1000+0.5) + } + } else { + a[$1] = $2 + } +} +END { + a["total_frames"]=int(a["duration"]*a["avg_frame_rate"]/1000/1000+0.5) + for (k in a) { + printf "\n", a[k], mark[k] + } +} diff --git a/ffmpeg-to-mkv-seektable.awk b/ffmpeg-to-mkv-seektable.awk new file mode 100644 index 0000000..6e31bc6 --- /dev/null +++ b/ffmpeg-to-mkv-seektable.awk @@ -0,0 +1,33 @@ +BEGIN { + RS="--\n" + ORS="\n" + FS="[=\n]" + frm=0 + offset=0 +} +# A typical record looks like: +# pkt_pts_time=166.725833 +# pkt_pos=596336 +# pict_type=I +# coded_picture_number=13 +# interlaced_frame=0 +# -- +{ + pkt_pts_time=int($2*1000+0.5) + pkt_pos=$4 + frame=$8+offset + if (frm==0) { + startpts=pkt_pts_time + # a bug? in ffprobe makes the frame numbers deficient by 2 after frame 0 + # but this does not seem to affect mkv containers made by ffmpeg? + if (ending != "mkv") { + offset=2 + } + } + frm=1 + pkt_pts_time=pkt_pts_time-startpts + # + # + print "" + print "" +} diff --git a/ffmpeg-to-mkv.sh b/ffmpeg-to-mkv.sh new file mode 100755 index 0000000..a62e41e --- /dev/null +++ b/ffmpeg-to-mkv.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -e +set -x + +file=${file:-"$A1"} +recordingsdir=${A2:-"/var/lib/mythtv/recordings"} +chanid=$1 +starttimeutc=$2 +title=${title:-"$A3"} +subtitle=${subtitle:-"$A4"} +season=${season:-"$A5"} +episode=${episode:-"$A6"} + +ending=$(echo $file | awk -v FS='.' '{print $NF}') +outfile=${file/%.${ending}/.mkv} +debug=${debug:-"eval"} +ffmpeg=${ffmpeg:-"/usr/local/bin/ffmpeg"} +destdir=${destdir:-"${recordingsdir/recordings/videos\/transcode}"} +#chanid=${file:0:4} +#starttimeutc=${file:5:13} +if test $chanid -eq 1031; then + lang="fre" +else + lang="en" +fi + +cmd="ionice -c 3 $ffmpeg -y -i ${recordingsdir}/${file} -f lavfi -i movie=${recordingsdir}/${file}[out0+subcc] -preset slow -crf 21 -map 0 -map -0:d -codec:a copy -map 1:s -map_metadata 1:s:0 -metadata:s:s \"language=${lang}\" -g 60 -keyint_min 30 -metadata \"title=${title}\" -metadata \"subtitle=${subtitle}\" -metadata \"season=${season}\" -metadata \"episode=${episode}\" -metadata \"service_name=$0\" ${destdir}/${outfile}" + +echo Information: +echo chanid: $chanid +echo starttimeutc: $starttimeutc +echo file: $file +echo outfile: $outfile +echo recordingsdir: $recordingsdir +echo title: $title +echo subtitle: $subtitle +echo season: $season +echo episode: $episode +echo ending: $ending +echo debug: $debug +echo ffmpeg: $ffmpeg +echo destdir: $destdir +echo cmd: $cmd + +$debug $cmd +exitstatus=$? + +if ! test $exitstatus -eq 0; then + exit $exitstatus +fi +touch --reference="${recordingsdir}/${file}" "${destdir}/${outfile}" +myth-update-basename.sh $chanid $starttimeutc $outfile +myth-rebuild-seektable.sh $chanid $starttimeutc + +exit $? diff --git a/myth-rebuild-seektable.sh b/myth-rebuild-seektable.sh new file mode 100755 index 0000000..86c63d4 --- /dev/null +++ b/myth-rebuild-seektable.sh @@ -0,0 +1,109 @@ +#!/bin/bash +set -e +set -x +chanid=$1 +starttime=$2 +if test -z "$3"; then + workdir=${workdir:-"/tmp"} +else + workdir=$3 +fi +fileroot="${chanid}_${starttime}" +ffprobe=${ffprobe:-"ionice -c 3 ffprobe"} +ffmpeg_to_mkv_seektable_awk=${ffmpeg_to_mkv_seektable_awk:-"/usr/local/bin/ffmpeg-to-mkv-seektable.awk"} +ffmpeg_to_mkv_marks_awk=${ffmpeg_to_mkv_marks_awk:-"/usr/local/bin/ffmpeg-to-mkv-marks.awk"} +# testing only +destdir=${destdir:-"/var/lib/mythtv/recordings"} +outfile=${outfile:-"$(ls $destdir/${fileroot}.{mpg,ts,mkv,mp4} 2>/dev/null | sort -u | head -n 1)"} +ending=$(echo "$outfile" | awk -v FS='.' '{print $NF}') + +if test -z "$outfile"; then + echo No recording file found with root $fileroot and ending mpg, ts, mkv or mp4 + exit 7 +fi +if ! test -r "$outfile"; then + echo Recording file $outfile is not readable. + exit 8 +else + outfile="$(basename $outfile)" +fi +if test -z "$ending"; then + echo Did not find file ending. Stop. + exit 9 +fi + +function get_keyframe_data { + key_frame_data=$workdir/${outfile}.2.txt + $ffprobe -v error -select_streams v:0 -show_entries frame=pict_type,coded_picture_number,pkt_pos,pkt_pts_time,interlaced_frame:side_data=nil -print_format default=noprint_wrappers=1 -i ${destdir}/${outfile} | grep --context=2 'pict_type=I' | awk -v ending="${ending}" -f $ffmpeg_to_mkv_seektable_awk > $key_frame_data + + exitstatus=$? + if ! test $exitstatus -eq 0; then + exit $exitstatus + fi + if ! test -r $key_frame_data; then + exit 5 + fi +} +function get_markup_data { + markup_data=$workdir/${outfile}.3.txt + $ffprobe -v error -show_format -show_streams -select_streams v:0 -show_entries stream=width,height,avg_frame_rate:format=duration -print_format default -i ${destdir}/${outfile} | awk -f $ffmpeg_to_mkv_marks_awk > $markup_data + + exitstatus=$? + if ! test $exitstatus -eq 0; then + exit $exitstatus + fi + if ! test -r $markup_data; then + exit 5 + fi +} +function set_markup_xml { + get_keyframe_data + get_markup_data + markup_xml=$workdir/${outfile}.xml +cat < $markup_xml + + + + +EOF +cat $key_frame_data $markup_data >> $markup_xml +cat <> $markup_xml + + + +EOF + exitstatus=$? + if ! test $exitstatus -eq 0; then + exit $exitstatus + fi + if ! test -r $markup_data; then + exit 5 + fi +} +function rebuild_seektable { + if test "$ending" = "mpg" -o "$ending" = "ts"; then + mythcommflag --chanid $chanid --starttime $starttime --rebuild + else + # file is mkv or mp4, so use ffprobe and mythutil + set_markup_xml && mythutil --quiet --chanid $chanid --starttime $starttime --setmarkup $markup_xml + # need to update filesize, too, since markup does not contain this + myth-update-basename.sh $chanid $starttime $outfile + fi +} + +for t in skiplist cutlist seektable; do + mythutil --chanid $chanid --starttime $starttime --clear$t +done +rebuild_seektable + + +# function clear_seeks_from_recordedmarkup { +# markupfile="$workdir/$1.xml" +# mythutil --quiet --chanid $chanid --starttime $starttime --getmarkup $markupfile && sed -ri -e '/seek/{d}; /mark/{/type="3[0-2]|1[0-9]"/{p}; d};' $markupfile && mythutil --quiet --chanid $chanid --starttime $starttime --setmarkup $markupfile +# } +# function remove_start_prog { +# markupfile="$workdir/$1.xml" +# mythutil --quiet --chanid $chanid --starttime $starttime --getmarkup $markupfile && sed -ri -e '/type="40"/{d}' $markupfile && mythutil --quiet --chanid $chanid --starttime $starttime --setmarkup $markupfile +# } + +exit $? diff --git a/myth-update-basename.sh b/myth-update-basename.sh new file mode 100755 index 0000000..7f87066 --- /dev/null +++ b/myth-update-basename.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +set -x +chanid=$1 +starttimeutc=$2 +basename=$3 +filesize=$(stat --printf="%s" $destdir/$basename) +dir="$HOME/.mythtv/" +config="config.xml" +mysql_password=$(awk -v FS='[<>]' '/Password/{print $3}' $dir/$config) +printf "update recorded set basename='%s', filesize='%s' where chanid=%s and starttime=%s;" $basename $filesize $chanid $starttimeutc | mysql --user=${USER} --password=${mysql_password} mythconverg +