working PGO

Jenkinsfile

@@ -39,14 +39,16 @@ pipeline {
         stage('build ffmpeg on darwin') {
             matrix {
                 axes {
-                    axis { name 'OPT_LTO'; values 'OPT=0 LTO=OFF', 'OPT=3 LTO=ON' }
-                    axis { name 'STATIC'; values 'ON', 'OFF' }
+                    axis { name 'COMP_OPTS'; values
+                        'OPT=0 LTO=OFF STATIC=OFF',
+                        'OPT=2 LTO=OFF STATIC=ON',
+                        'OPT=3 LTO=ON STATIC=ON PGO=ON' }
                 }
                 stages {
                     stage('build on darwin ') {
                         agent { label "darwin" }
                         steps {
-                            sh "${OPT_LTO} ./scripts/build.sh"
+                            sh "${COMP_OPTS} ./scripts/build.sh"
                             archiveArtifacts allowEmptyArchive: true, artifacts: 'gitignore/package/*.tar.xz', defaultExcludes: false
                         }
                     }
@@ -58,15 +60,17 @@ pipeline {
                 axes {
                     axis { name 'ARCH'; values 'armv8-a', 'x86-64-v3' }
                     axis { name 'DISTRO'; values 'ubuntu', 'fedora', 'debian', 'archlinux' }
-                    axis { name 'OPT_LTO'; values 'OPT=0 LTO=OFF', 'OPT=3 LTO=ON' }
-                    axis { name 'STATIC'; values 'ON', 'OFF' }
+                    axis { name 'COMP_OPTS'; values
+                        'OPT=0 LTO=OFF STATIC=OFF',
+                        'OPT=2 LTO=OFF STATIC=ON',
+                        'OPT=3 LTO=ON STATIC=ON PGO=ON' }
                 }
                 stages {
                     stage('build ffmpeg on linux using docker') {
                         agent { label "linux && ${ARCH}" }
                         steps {
                             withDockerCreds {
-                                sh "${OPT_LTO} ./scripts/build_with_docker.sh ${DISTRO}"
+                                sh "${COMP_OPTS} ./scripts/build_with_docker.sh ${DISTRO}"
                                 archiveArtifacts allowEmptyArchive: true, artifacts: 'gitignore/package/*.tar.xz', defaultExcludes: false
                             }
                         }

lib/build.sh

@@ -101,8 +101,7 @@ set_compile_opts() {
 		fi
 		CMAKE_FLAGS+=("-DCMAKE_LINKER=${USE_LD}")
 		local compilerDir="${LOCAL_PREFIX}/compiler-tools"
-		test -d "${compilerDir}" && rm -rf "${compilerDir}"
-		mkdir -p "${compilerDir}"
+		recreate_dir "${compilerDir}" || return 1
 		# real:gnu:clang:generic
 		local compilerMap="\
 ${CC}:gcc:clang:cc
@@ -166,6 +165,20 @@ exec \"${realT}\" ${addFlag} \"\$@\"" >"${compilerDir}/${genericT}"
 	# TODO use cygpath for windows
 	CPPFLAGS_ARR+=("-I${PREFIX}/include")
 
+	# if PGO is enabled, first build run will be to generate
+	# second run will be to use generated profdata
+	if [[ ${PGO} == 'ON' ]]; then
+		if [[ ${PGO_RUN} == 'generate' ]]; then
+			local pgoFlag="-fprofile-generate"
+			CFLAGS_ARR+=("${pgoFlag}")
+			LDFLAGS_ARR+=("${pgoFlag}")
+		else
+			local pgoFlag="-fprofile-use=${PGO_PROFDATA}"
+			CFLAGS_ARR+=("${pgoFlag}")
+			LDFLAGS_ARR+=("${pgoFlag}")
+		fi
+	fi
+
 	# enabling link-time optimization
 	if [[ ${LTO} == 'ON' ]]; then
 		LTO_FLAG='-flto=full'
@@ -539,6 +552,14 @@ FB_FUNC_NAMES+=('build')
 # shellcheck disable=SC2034
 FB_FUNC_DESCS['build']='build ffmpeg with the desired configuration'
 build() {
+	# if PGO is enabled, build will call build
+	# only want to recursively build on the first run
+	if [[ ${PGO} == 'ON' && ${PGO_RUN} != 'generate' ]]; then
+		PGO_RUN='generate' build || return 1
+		# will need to reset compile opts
+		unset FB_COMPILE_OPTS_SET
+	fi
+
 	set_compile_opts || return 1
 
 	for build in ${ENABLE}; do
@@ -547,9 +568,16 @@ build() {
 		unset REQUIRES_REBUILD
 	done
 	do_build ffmpeg || return 1
+
+	# skip packaging on PGO generate run
+	if [[ ${PGO} == 'ON' && ${PGO_RUN} == 'generate' ]]; then
+		PATH="${PREFIX}/bin:${PATH}" gen_profdata
+		return $?
+	fi
+
 	local ffmpegBin="${PREFIX}/bin/ffmpeg"
 	# run ffmpeg to show completion
-	"${ffmpegBin}" -version
+	"${ffmpegBin}" -version || return 1
 
 	# suggestion for path
 	hash -r

lib/docker.sh

@@ -11,8 +11,7 @@ DOCKER_WORKDIR='/workdir'
 set_docker_run_flags() {
 	local cargo_git="${IGN_DIR}/cargo/git"
 	local cargo_registry="${IGN_DIR}/cargo/registry"
-	test -d "${cargo_git}" || mkdir -p "${cargo_git}"
-	test -d "${cargo_registry}" || mkdir -p "${cargo_registry}"
+	ensure_dir "${cargo_git}" "${cargo_registry}"
 	DOCKER_RUN_FLAGS=(
 		--rm
 		-v "${cargo_git}:/root/.cargo/git"

lib/efg.sh

@@ -35,13 +35,13 @@ set_efg_opts() {
 			sudo ln -sf "${SCRIPT_DIR}/efg.sh" \
 				"${EFG_INSTALL_PATH}" || return 1
 			echo_pass "succesfull install"
-			exit 0
+			return ${FUNC_EXIT_SUCCESS}
 			;;
 		U)
 			echo_warn "attempting uninstall"
 			sudo rm "${EFG_INSTALL_PATH}" || return 1
 			echo_pass "succesfull uninstall"
-			exit 0
+			return ${FUNC_EXIT_SUCCESS}
 			;;
 		i)
 			if [[ $# -lt 2 ]]; then
@@ -126,8 +126,7 @@ efg_segment() {
 	local segmentBitrates=()
 
 	# clean workspace
-	test -d "${EFG_DIR}" && rm -rf "${EFG_DIR}"
-	mkdir -p "${EFG_DIR}"
+	recreate_dir "${EFG_DIR}" || return 1
 
 	# split up video into segments based on start times
 	for ((time = 0; time < duration; time += timeBetweenSegments)); do
@@ -245,8 +244,14 @@ efg() {
 	# encode N highest-bitrate segments
 	ENCODE_SEGMENTS=5
 
-	set_efg_opts "$@" || return 1
-	test -d "${EFG_DIR}" || mkdir "${EFG_DIR}"
+	set_efg_opts "$@"
+	local ret=$?
+	if [[ ${ret} -eq ${FUNC_EXIT_SUCCESS} ]]; then
+		return 0
+	elif [[ ${ret} -ne 0 ]]; then
+		return ${ret}
+	fi
+	ensure_dir "${EFG_DIR}"
 
 	GRAIN_LOG="${EFG_DIR}/${LOW}-${STEP}-${HIGH}-grains.txt"
 

lib/encode.sh

@@ -99,11 +99,11 @@ get_encode_versions() {
 	# shellcheck disable=SC2155
 	local output="$(ffmpeg -version 2>&1)"
 	while read -r line; do
-		if line_contains "${line}" 'ffmpeg='; then
+		if line_starts_with "${line}" 'ffmpeg='; then
 			ffmpegVersion="${line}"
-		elif line_contains "${line}" 'libsvtav1_psy=' || line_contains "${line}" 'libsvtav1='; then
+		elif line_starts_with "${line}" 'libsvtav1'; then
 			videoEncVersion="${line}"
-		elif line_contains "${line}" 'libopus='; then
+		elif line_starts_with "${line}" 'libopus='; then
 			audioEncVersion="${line}"
 		fi
 	done <<<"${output}"
@@ -188,25 +188,25 @@ set_encode_opts() {
 	while getopts "${opts}" flag; do
 		case "${flag}" in
 		u)
-			encode_update
-			exit $?
+			encode_update || return 1
+			return ${FUNC_EXIT_SUCCESS}
 			;;
 		I)
 			echo_warn "attempting install"
 			sudo ln -sf "${SCRIPT_DIR}/encode.sh" \
 				"${ENCODE_INSTALL_PATH}" || return 1
 			echo_pass "succesfull install"
-			exit 0
+			return ${FUNC_EXIT_SUCCESS}
 			;;
 		U)
 			echo_warn "attempting uninstall"
 			sudo rm "${ENCODE_INSTALL_PATH}" || return 1
 			echo_pass "succesfull uninstall"
-			exit 0
+			return ${FUNC_EXIT_SUCCESS}
 			;;
 		v)
-			get_encode_versions print
-			exit $?
+			get_encode_versions print || return 1
+			return ${FUNC_EXIT_SUCCESS}
 			;;
 		i)
 			if [[ $# -lt 2 ]]; then
@@ -253,7 +253,7 @@ set_encode_opts() {
 			if ! is_positive_integer "${OPTARG}" || test ${OPTARG} -gt 63; then
 				echo_fail "${OPTARG} is not a valid CRF value (0-63)"
 				encode_usage
-				exit 1
+				return 1
 			fi
 			CRF="${OPTARG}"
 			OPTS_USED=$((OPTS_USED + 2))
@@ -471,6 +471,12 @@ FB_FUNC_NAMES+=('encode')
 # shellcheck disable=SC2034
 FB_FUNC_DESCS['encode']='encode a file using libsvtav1_psy and libopus'
 encode() {
-	set_encode_opts "$@" || return 1
+	set_encode_opts "$@"
+	local ret=$?
+	if [[ ${ret} -eq ${FUNC_EXIT_SUCCESS} ]]; then
+		return 0
+	elif [[ ${ret} -ne 0 ]]; then
+		return ${ret}
+	fi
 	gen_encode_script || return 1
 }

lib/ffmpeg.sh

@@ -126,3 +126,41 @@ get_stream_lang() {
 		-of default=noprint_wrappers=1:nokey=1 \
 		"${file}"
 }
+
+gen_video() {
+	local outFile="$1"
+	local addFlags=()
+	shift
+
+	local vf="format=yuv420p10le"
+	for arg in "$@"; do
+		case "${arg}" in
+		'1080p') resolution='1920x1080' ;;
+		'2160p') resolution='3840x2160' ;;
+		'grain=yes') vf+=",noise=alls=15:allf=t+u" ;;
+		'hdr=yes')
+			vf+=",setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc"
+			addFlags+=(
+				-color_primaries bt2020
+				-color_trc smpte2084
+				-colorspace bt2020nc
+				-metadata:s:v:0 "mastering_display_metadata=G(13250,34500)B(7500,3000)R(34000,16000)WP(15635,16450)L(10000000,1)"
+				-metadata:s:v:0 "content_light_level=1000,400"
+			)
+			;;
+		*) echo_fail "bad arg ${arg}" && return 1 ;;
+		esac
+	done
+
+	ffmpeg -y \
+		-hide_banner \
+		-f lavfi \
+		-i "testsrc2=size=${resolution}:rate=24:duration=5" \
+		-vf "${vf}" \
+		-c:v ffv1 \
+		-level 3 \
+		-g 1 \
+		-color_range tv \
+		"${addFlags[@]}" \
+		"${outFile}"
+}

lib/package.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
 check_for_package_cfg() {
-	local requiredCfg='ON:ON:3'
-	local currentCfg="${STATIC}:${LTO}:${OPT}"
+	local requiredCfg='ON:ON:ON:3'
+	local currentCfg="${STATIC}:${LTO}:${PGO}:${OPT}"
 	if [[ ${currentCfg} == "${requiredCfg}" ]]; then
 		return 0
 	else
@@ -14,12 +14,10 @@ FB_FUNC_NAMES+=('package')
 FB_FUNC_DESCS['package']='package ffmpeg build'
 package() {
 	local pkgDir="${IGN_DIR}/package"
-	test -d "${pkgDir}" && rm -rf "${pkgDir}"
+	recreate_dir "${pkgDir}" || return 1
 	check_for_package_cfg || return 0
 
 	echo_info "packaging"
-	mkdir "${pkgDir}" || return 1
-
 	set_compile_opts || return 1
 	cp "${PREFIX}/bin/ff"* "${pkgDir}/"
 

lib/pgo.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+PGO_DIR="${IGN_DIR}/pgo"
+PGO_PROFDATA="${PGO_DIR}/prof.profdata"
+gen_profdata() {
+	recreate_dir "${PGO_DIR}" || return 1
+	cd "${PGO_DIR}" || return 1
+	setup_pgo_clips || return 1
+	for vid in *.mkv; do
+		local args=()
+		# add precalculated grain amount based off of filename
+		line_contains "${vid}" 'grain' && args+=(-g 16)
+		# make fhd preset 2
+		line_contains "${vid}" 'fhd' && args+=(-P 2)
+
+		LLVM_PROFILE_FILE="${PGO_DIR}/default_%p.profraw" \
+			encode -i "${vid}" "${args[@]}" "encoded-${vid}" || return 1
+	done
+
+	# merge profraw into profdata
+	local mergeCmd=()
+	# darwin needs special invoke
+	if is_darwin; then
+		mergeCmd+=(xcrun)
+	fi
+
+	mergeCmd+=(
+		llvm-profdata
+		merge
+		"--output=${PGO_PROFDATA}"
+	)
+	"${mergeCmd[@]}" default*.profraw || return 1
+
+	return 0
+}
+
+setup_pgo_clips() {
+	local clips=(
+		"fhd-grainy.mkv 1080p,grain=yes"
+		"uhd.mkv 2160p"
+		"uhd-hdr.mkv 2160p,hdr=yes"
+	)
+	for clip in "${clips[@]}"; do
+		local genVid genVidArgs pgoFile genVidArgsArr
+		IFS=' ' read -r genVid genVidArgs <<<"${clip}"
+		# pgo path is separate
+		pgoFile="${PGO_DIR}/${genVid}"
+		genVid="${TMP_DIR}/${genVid}"
+		# create array of args split with ,
+		genVidArgsArr=(${genVidArgs//,/ })
+		# create generated vid without any profiling if needed
+		test -f "${genVid}" ||
+			LLVM_PROFILE_FILE='/dev/null' gen_video "${genVid}" "${genVidArgsArr[@]}" || return 1
+		# and move to the pgo directory
+		test -f "${pgoFile}" ||
+			cp "${genVid}" "${pgoFile}" || return 1
+
+	done
+}

lib/utils.sh

@@ -366,3 +366,18 @@ get_pkgconfig_version() {
 	local pkg="$1"
 	pkg-config --modversion "${pkg}"
 }
+
+recreate_dir() {
+	local dirs=("$@")
+	for dir in "${dirs[@]}"; do
+		test -d "${dir}" && rm -rf "${dir}"
+		mkdir -p "${dir}" || return 1
+	done
+}
+
+ensure_dir() {
+	local dirs=("$@")
+	for dir in "${dirs[@]}"; do
+		test -d "${dir}" || mkdir -p "${dir}" || return 1
+	done
+}

main.sh

@@ -15,6 +15,11 @@ DOCKER_DIR="${IGN_DIR}/docker"
 PATCHES_DIR="${REPO_DIR}/patches"
 export REPO_DIR IGN_DIR TMP_DIR DL_DIR BUILD_DIR CCACHE_DIR DOCKER_DIR PATCHES_DIR
 
+# some functions need a way to signal early
+# returns instead of failures, so if a function
+# returns ${FUNC_EXIT_SUCCESS}, stop processing
+test -v FUNC_EXIT_SUCCESS || readonly FUNC_EXIT_SUCCESS=9
+
 # make paths if needed
 IGN_DIRS=(
 	"${TMP_DIR}"