diff --git a/Jenkinsfile b/Jenkinsfile index 459f4ae..aedcf57 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,14 +39,16 @@ pipeline { stage('build ffmpeg on darwin') { matrix { axes { - axis { name 'OPT_LTO'; values 'OPT=0 LTO=OFF', 'OPT=3 LTO=ON' } - axis { name 'STATIC'; values 'ON', 'OFF' } + axis { name 'COMP_OPTS'; values + 'OPT=0 LTO=OFF STATIC=OFF', + 'OPT=2 LTO=OFF STATIC=ON', + 'OPT=3 LTO=ON STATIC=ON PGO=ON' } } stages { stage('build on darwin ') { agent { label "darwin" } steps { - sh "${OPT_LTO} ./scripts/build.sh" + sh "${COMP_OPTS} ./scripts/build.sh" archiveArtifacts allowEmptyArchive: true, artifacts: 'gitignore/package/*.tar.xz', defaultExcludes: false } } @@ -58,15 +60,17 @@ pipeline { axes { axis { name 'ARCH'; values 'armv8-a', 'x86-64-v3' } axis { name 'DISTRO'; values 'ubuntu', 'fedora', 'debian', 'archlinux' } - axis { name 'OPT_LTO'; values 'OPT=0 LTO=OFF', 'OPT=3 LTO=ON' } - axis { name 'STATIC'; values 'ON', 'OFF' } + axis { name 'COMP_OPTS'; values + 'OPT=0 LTO=OFF STATIC=OFF', + 'OPT=2 LTO=OFF STATIC=ON', + 'OPT=3 LTO=ON STATIC=ON PGO=ON' } } stages { stage('build ffmpeg on linux using docker') { agent { label "linux && ${ARCH}" } steps { withDockerCreds { - sh "${OPT_LTO} ./scripts/build_with_docker.sh ${DISTRO}" + sh "${COMP_OPTS} ./scripts/build_with_docker.sh ${DISTRO}" archiveArtifacts allowEmptyArchive: true, artifacts: 'gitignore/package/*.tar.xz', defaultExcludes: false } } diff --git a/lib/build.sh b/lib/build.sh index 28659e9..54c22c6 100644 --- a/lib/build.sh +++ b/lib/build.sh @@ -101,8 +101,7 @@ set_compile_opts() { fi CMAKE_FLAGS+=("-DCMAKE_LINKER=${USE_LD}") local compilerDir="${LOCAL_PREFIX}/compiler-tools" - test -d "${compilerDir}" && rm -rf "${compilerDir}" - mkdir -p "${compilerDir}" + recreate_dir "${compilerDir}" || return 1 # real:gnu:clang:generic local compilerMap="\ ${CC}:gcc:clang:cc @@ -166,6 +165,20 @@ exec \"${realT}\" ${addFlag} \"\$@\"" >"${compilerDir}/${genericT}" # TODO use cygpath for windows CPPFLAGS_ARR+=("-I${PREFIX}/include") + # if PGO is enabled, first build run will be to generate + # second run will be to use generated profdata + if [[ ${PGO} == 'ON' ]]; then + if [[ ${PGO_RUN} == 'generate' ]]; then + local pgoFlag="-fprofile-generate" + CFLAGS_ARR+=("${pgoFlag}") + LDFLAGS_ARR+=("${pgoFlag}") + else + local pgoFlag="-fprofile-use=${PGO_PROFDATA}" + CFLAGS_ARR+=("${pgoFlag}") + LDFLAGS_ARR+=("${pgoFlag}") + fi + fi + # enabling link-time optimization if [[ ${LTO} == 'ON' ]]; then LTO_FLAG='-flto=full' @@ -539,6 +552,14 @@ FB_FUNC_NAMES+=('build') # shellcheck disable=SC2034 FB_FUNC_DESCS['build']='build ffmpeg with the desired configuration' build() { + # if PGO is enabled, build will call build + # only want to recursively build on the first run + if [[ ${PGO} == 'ON' && ${PGO_RUN} != 'generate' ]]; then + PGO_RUN='generate' build || return 1 + # will need to reset compile opts + unset FB_COMPILE_OPTS_SET + fi + set_compile_opts || return 1 for build in ${ENABLE}; do @@ -547,9 +568,16 @@ build() { unset REQUIRES_REBUILD done do_build ffmpeg || return 1 + + # skip packaging on PGO generate run + if [[ ${PGO} == 'ON' && ${PGO_RUN} == 'generate' ]]; then + PATH="${PREFIX}/bin:${PATH}" gen_profdata + return $? + fi + local ffmpegBin="${PREFIX}/bin/ffmpeg" # run ffmpeg to show completion - "${ffmpegBin}" -version + "${ffmpegBin}" -version || return 1 # suggestion for path hash -r diff --git a/lib/docker.sh b/lib/docker.sh index ff375d9..b83f65b 100644 --- a/lib/docker.sh +++ b/lib/docker.sh @@ -11,8 +11,7 @@ DOCKER_WORKDIR='/workdir' set_docker_run_flags() { local cargo_git="${IGN_DIR}/cargo/git" local cargo_registry="${IGN_DIR}/cargo/registry" - test -d "${cargo_git}" || mkdir -p "${cargo_git}" - test -d "${cargo_registry}" || mkdir -p "${cargo_registry}" + ensure_dir "${cargo_git}" "${cargo_registry}" DOCKER_RUN_FLAGS=( --rm -v "${cargo_git}:/root/.cargo/git" diff --git a/lib/efg.sh b/lib/efg.sh index 6a0c3c2..3379880 100644 --- a/lib/efg.sh +++ b/lib/efg.sh @@ -35,13 +35,13 @@ set_efg_opts() { sudo ln -sf "${SCRIPT_DIR}/efg.sh" \ "${EFG_INSTALL_PATH}" || return 1 echo_pass "succesfull install" - exit 0 + return ${FUNC_EXIT_SUCCESS} ;; U) echo_warn "attempting uninstall" sudo rm "${EFG_INSTALL_PATH}" || return 1 echo_pass "succesfull uninstall" - exit 0 + return ${FUNC_EXIT_SUCCESS} ;; i) if [[ $# -lt 2 ]]; then @@ -126,8 +126,7 @@ efg_segment() { local segmentBitrates=() # clean workspace - test -d "${EFG_DIR}" && rm -rf "${EFG_DIR}" - mkdir -p "${EFG_DIR}" + recreate_dir "${EFG_DIR}" || return 1 # split up video into segments based on start times for ((time = 0; time < duration; time += timeBetweenSegments)); do @@ -245,8 +244,14 @@ efg() { # encode N highest-bitrate segments ENCODE_SEGMENTS=5 - set_efg_opts "$@" || return 1 - test -d "${EFG_DIR}" || mkdir "${EFG_DIR}" + set_efg_opts "$@" + local ret=$? + if [[ ${ret} -eq ${FUNC_EXIT_SUCCESS} ]]; then + return 0 + elif [[ ${ret} -ne 0 ]]; then + return ${ret} + fi + ensure_dir "${EFG_DIR}" GRAIN_LOG="${EFG_DIR}/${LOW}-${STEP}-${HIGH}-grains.txt" diff --git a/lib/encode.sh b/lib/encode.sh index 8ea7ed0..a38d4e4 100644 --- a/lib/encode.sh +++ b/lib/encode.sh @@ -99,11 +99,11 @@ get_encode_versions() { # shellcheck disable=SC2155 local output="$(ffmpeg -version 2>&1)" while read -r line; do - if line_contains "${line}" 'ffmpeg='; then + if line_starts_with "${line}" 'ffmpeg='; then ffmpegVersion="${line}" - elif line_contains "${line}" 'libsvtav1_psy=' || line_contains "${line}" 'libsvtav1='; then + elif line_starts_with "${line}" 'libsvtav1'; then videoEncVersion="${line}" - elif line_contains "${line}" 'libopus='; then + elif line_starts_with "${line}" 'libopus='; then audioEncVersion="${line}" fi done <<<"${output}" @@ -188,25 +188,25 @@ set_encode_opts() { while getopts "${opts}" flag; do case "${flag}" in u) - encode_update - exit $? + encode_update || return 1 + return ${FUNC_EXIT_SUCCESS} ;; I) echo_warn "attempting install" sudo ln -sf "${SCRIPT_DIR}/encode.sh" \ "${ENCODE_INSTALL_PATH}" || return 1 echo_pass "succesfull install" - exit 0 + return ${FUNC_EXIT_SUCCESS} ;; U) echo_warn "attempting uninstall" sudo rm "${ENCODE_INSTALL_PATH}" || return 1 echo_pass "succesfull uninstall" - exit 0 + return ${FUNC_EXIT_SUCCESS} ;; v) - get_encode_versions print - exit $? + get_encode_versions print || return 1 + return ${FUNC_EXIT_SUCCESS} ;; i) if [[ $# -lt 2 ]]; then @@ -253,7 +253,7 @@ set_encode_opts() { if ! is_positive_integer "${OPTARG}" || test ${OPTARG} -gt 63; then echo_fail "${OPTARG} is not a valid CRF value (0-63)" encode_usage - exit 1 + return 1 fi CRF="${OPTARG}" OPTS_USED=$((OPTS_USED + 2)) @@ -471,6 +471,12 @@ FB_FUNC_NAMES+=('encode') # shellcheck disable=SC2034 FB_FUNC_DESCS['encode']='encode a file using libsvtav1_psy and libopus' encode() { - set_encode_opts "$@" || return 1 + set_encode_opts "$@" + local ret=$? + if [[ ${ret} -eq ${FUNC_EXIT_SUCCESS} ]]; then + return 0 + elif [[ ${ret} -ne 0 ]]; then + return ${ret} + fi gen_encode_script || return 1 } diff --git a/lib/ffmpeg.sh b/lib/ffmpeg.sh index 068389e..43a9d36 100644 --- a/lib/ffmpeg.sh +++ b/lib/ffmpeg.sh @@ -126,3 +126,41 @@ get_stream_lang() { -of default=noprint_wrappers=1:nokey=1 \ "${file}" } + +gen_video() { + local outFile="$1" + local addFlags=() + shift + + local vf="format=yuv420p10le" + for arg in "$@"; do + case "${arg}" in + '1080p') resolution='1920x1080' ;; + '2160p') resolution='3840x2160' ;; + 'grain=yes') vf+=",noise=alls=15:allf=t+u" ;; + 'hdr=yes') + vf+=",setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc" + addFlags+=( + -color_primaries bt2020 + -color_trc smpte2084 + -colorspace bt2020nc + -metadata:s:v:0 "mastering_display_metadata=G(13250,34500)B(7500,3000)R(34000,16000)WP(15635,16450)L(10000000,1)" + -metadata:s:v:0 "content_light_level=1000,400" + ) + ;; + *) echo_fail "bad arg ${arg}" && return 1 ;; + esac + done + + ffmpeg -y \ + -hide_banner \ + -f lavfi \ + -i "testsrc2=size=${resolution}:rate=24:duration=5" \ + -vf "${vf}" \ + -c:v ffv1 \ + -level 3 \ + -g 1 \ + -color_range tv \ + "${addFlags[@]}" \ + "${outFile}" +} diff --git a/lib/package.sh b/lib/package.sh index 0781457..cf34a50 100644 --- a/lib/package.sh +++ b/lib/package.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash check_for_package_cfg() { - local requiredCfg='ON:ON:3' - local currentCfg="${STATIC}:${LTO}:${OPT}" + local requiredCfg='ON:ON:ON:3' + local currentCfg="${STATIC}:${LTO}:${PGO}:${OPT}" if [[ ${currentCfg} == "${requiredCfg}" ]]; then return 0 else @@ -14,12 +14,10 @@ FB_FUNC_NAMES+=('package') FB_FUNC_DESCS['package']='package ffmpeg build' package() { local pkgDir="${IGN_DIR}/package" - test -d "${pkgDir}" && rm -rf "${pkgDir}" + recreate_dir "${pkgDir}" || return 1 check_for_package_cfg || return 0 echo_info "packaging" - mkdir "${pkgDir}" || return 1 - set_compile_opts || return 1 cp "${PREFIX}/bin/ff"* "${pkgDir}/" diff --git a/lib/pgo.sh b/lib/pgo.sh new file mode 100644 index 0000000..1c6212f --- /dev/null +++ b/lib/pgo.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +PGO_DIR="${IGN_DIR}/pgo" +PGO_PROFDATA="${PGO_DIR}/prof.profdata" +gen_profdata() { + recreate_dir "${PGO_DIR}" || return 1 + cd "${PGO_DIR}" || return 1 + setup_pgo_clips || return 1 + for vid in *.mkv; do + local args=() + # add precalculated grain amount based off of filename + line_contains "${vid}" 'grain' && args+=(-g 16) + # make fhd preset 2 + line_contains "${vid}" 'fhd' && args+=(-P 2) + + LLVM_PROFILE_FILE="${PGO_DIR}/default_%p.profraw" \ + encode -i "${vid}" "${args[@]}" "encoded-${vid}" || return 1 + done + + # merge profraw into profdata + local mergeCmd=() + # darwin needs special invoke + if is_darwin; then + mergeCmd+=(xcrun) + fi + + mergeCmd+=( + llvm-profdata + merge + "--output=${PGO_PROFDATA}" + ) + "${mergeCmd[@]}" default*.profraw || return 1 + + return 0 +} + +setup_pgo_clips() { + local clips=( + "fhd-grainy.mkv 1080p,grain=yes" + "uhd.mkv 2160p" + "uhd-hdr.mkv 2160p,hdr=yes" + ) + for clip in "${clips[@]}"; do + local genVid genVidArgs pgoFile genVidArgsArr + IFS=' ' read -r genVid genVidArgs <<<"${clip}" + # pgo path is separate + pgoFile="${PGO_DIR}/${genVid}" + genVid="${TMP_DIR}/${genVid}" + # create array of args split with , + genVidArgsArr=(${genVidArgs//,/ }) + # create generated vid without any profiling if needed + test -f "${genVid}" || + LLVM_PROFILE_FILE='/dev/null' gen_video "${genVid}" "${genVidArgsArr[@]}" || return 1 + # and move to the pgo directory + test -f "${pgoFile}" || + cp "${genVid}" "${pgoFile}" || return 1 + + done +} diff --git a/lib/utils.sh b/lib/utils.sh index f12b102..2b2fd5f 100644 --- a/lib/utils.sh +++ b/lib/utils.sh @@ -366,3 +366,18 @@ get_pkgconfig_version() { local pkg="$1" pkg-config --modversion "${pkg}" } + +recreate_dir() { + local dirs=("$@") + for dir in "${dirs[@]}"; do + test -d "${dir}" && rm -rf "${dir}" + mkdir -p "${dir}" || return 1 + done +} + +ensure_dir() { + local dirs=("$@") + for dir in "${dirs[@]}"; do + test -d "${dir}" || mkdir -p "${dir}" || return 1 + done +} diff --git a/main.sh b/main.sh index c1354f2..272de43 100755 --- a/main.sh +++ b/main.sh @@ -15,6 +15,11 @@ DOCKER_DIR="${IGN_DIR}/docker" PATCHES_DIR="${REPO_DIR}/patches" export REPO_DIR IGN_DIR TMP_DIR DL_DIR BUILD_DIR CCACHE_DIR DOCKER_DIR PATCHES_DIR +# some functions need a way to signal early +# returns instead of failures, so if a function +# returns ${FUNC_EXIT_SUCCESS}, stop processing +test -v FUNC_EXIT_SUCCESS || readonly FUNC_EXIT_SUCCESS=9 + # make paths if needed IGN_DIRS=( "${TMP_DIR}"