wip fixups

cfgs/builds.json

@@ -14,6 +14,11 @@
         "ext": "tar.gz",
         "url": "https://github.com/quietvoid/dovi_tool/archive/refs/tags/${ver}.${ext}"
     },
+    "libsvtav1": {
+        "ver": "3.0.2",
+        "ext": "tar.gz",
+        "url": "https://gitlab.com/AOMediaCodec/SVT-AV1/-/archive/v${ver}/SVT-AV1-v${ver}.${ext}"
+    },
     "libsvtav1_psy": {
         "ver": "3.0.2",
         "ext": "tar.gz",
@@ -22,6 +27,21 @@
             "dovi_tool", "hdr10plus_tool"
         ]
     },
+    "librav1e": {
+        "ver": "0.7.1",
+        "ext": "tar.gz",
+        "url": "https://github.com/xiph/rav1e/archive/refs/tags/v${ver}.${ext}"
+    },
+    "libaom": {
+        "ver": "3.12.1",
+        "ext": "tar.gz",
+        "url": "https://storage.googleapis.com/aom-releases/libaom-${ver}.${ext}"
+    },
+    "libvmaf": {
+        "ver": "3.0.0",
+        "ext": "tar.gz",
+        "url": "https://github.com/Netflix/vmaf/archive/refs/tags/v${ver}.${ext}"
+    },
     "libopus": {
         "ver": "1.5.2",
         "ext": "tar.gz",

cfgs/compile_opts.json

@@ -1,7 +1,7 @@
 {
     "clean": true,
-    "lto": true,
+    "lto": false,
-    "optimization": 3,
+    "optimization": 0,
     "static": true,
     "shared": false,
     "cpu": "native",

main.sh

@@ -2,10 +2,11 @@
 
 REPO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
 IGN_DIR="${REPO_DIR}/gitignore"
+TMP_DIR="${IGN_DIR}/tmp"
 DL_DIR="${IGN_DIR}/downloads"
 BUILD_DIR="${IGN_DIR}/builds"
 CCACHE_DIR="${IGN_DIR}/ccache"
-export REPO_DIR IGN_DIR DL_DIR BUILD_DIR CCACHE_DIR
+export REPO_DIR IGN_DIR TMP_DIR DL_DIR BUILD_DIR CCACHE_DIR
 
 # function names, descriptions, completions
 FB_FUNC_NAMES=()

scripts/build.sh

@@ -18,8 +18,10 @@ set_compile_opts() {
     JOBS="$(nproc)"
     export JOBS
     
-    machine="$(cc -dumpmachine)"
+    MACHINE="$(cc -dumpmachine)"
-    test "${machine}" != '' || return 1
+    test "${MACHINE}" != '' || return 1
+    export MACHINE
+    MACHINE_LIB="${PREFIX}/lib/${MACHINE}"
 
     # set prefix flags
     CONFIGURE_FLAGS+=("--prefix=${PREFIX}")
@@ -27,7 +29,7 @@ set_compile_opts() {
     CMAKE_FLAGS+=("-DCMAKE_PREFIX_PATH=${PREFIX}")
     CMAKE_FLAGS+=("-DCMAKE_INSTALL_PREFIX=${PREFIX}")
     PKG_CONFIG_PATH="${PREFIX}/lib/pkgconfig:${PKG_CONFIG_PATH}"
-    PKG_CONFIG_PATH="${PREFIX}/lib/${machine}/pkgconfig:${PKG_CONFIG_PATH}"
+    PKG_CONFIG_PATH="${MACHINE_LIB}/pkgconfig:${PKG_CONFIG_PATH}"
     echo_info "PKG_CONFIG_PATH = ${PKG_CONFIG_PATH}"
 
     # add prefix include 
@@ -49,15 +51,15 @@ set_compile_opts() {
         echo_info "building with LTO"
         LTO_SWITCH='ON'
         LTO_FLAG='-flto'
-        LTO_BOOL='true'
         C_FLAGS+=("${LTO_FLAG}")
         CONFIGURE_FLAGS+=('--enable-lto')
+        MESON_FLAGS+=("-Db_lto=true")
         RUSTFLAGS+=("-C lto=yes" "-C inline-threshold=1000" "-C codegen-units=1")
     else
         echo_info "building without LTO"
         LTO_SWITCH='OFF'
         LTO_FLAG=''
-        LTO_BOOL='false'
+        MESON_FLAGS+=("-Db_lto=false")
         RUSTFLAGS+=("-C lto=no")
     fi
 
@@ -72,20 +74,29 @@ set_compile_opts() {
     echo_info "building with optimization: ${OPT_LVL}"
     
     # static/shared linking
-    unset PKG_CFG_FLAGS
+    unset PKG_CFG_FLAGS LIB_SUFF
-    export PKG_CFG_FLAGS
+    export PKG_CFG_FLAGS LIB_SUFF
-    if test "$(jq .static "${COMPILE_CFG}")" == 'true'; then
+    isStatic="$(test "$(jq .static "${COMPILE_CFG}")" == 'true' ; echo $?)"
-        LDFLAGS+='-static'
+    isShared="$(test "$(jq .shared "${COMPILE_CFG}")" == 'true' ; echo $?)"
+    if test $((isStatic + isShared)) -eq 2; then
+        echo_exit "Cannot have static and shared compile options"
+    fi
+    if test "${isStatic}" -eq 0; then
+        LDFLAGS+=('-static')
         CONFIGURE_FLAGS+=('--enable-static')
         CMAKE_FLAGS+=("-DBUILD_SHARED_LIBS=OFF")
         MESON_FLAGS+=('--default-library=static')
+        CMAKE_FLAGS+=("-DCMAKE_EXE_LINKER_FLAGS='-static'")
         PKG_CFG_FLAGS='--static'
+        LIB_SUFF='a'
     fi
-    if test "$(jq .shared "${COMPILE_CFG}")" == 'true'; then
+    if test "${isShared}" -eq 0; then
+        LDFLAGS+=("-Wl,-rpath,${MACHINE_LIB}")
         CONFIGURE_FLAGS+=('--enable-shared')
         CMAKE_FLAGS+=("-DBUILD_SHARED_LIBS=ON")
-        CMAKE_FLAGS+=("-DCMAKE_INSTALL_RPATH=${PREFIX}/lib;${PREFIX}/lib/${machine}")
+        CMAKE_FLAGS+=("-DCMAKE_INSTALL_RPATH=${PREFIX}/lib;${MACHINE_LIB}")        
         FFMPEG_EXTRA_FLAGS+=('--enable-rpath')
+        LIB_SUFF='so'
     fi
 
     # architecture/cpu compile flags
@@ -110,10 +121,12 @@ set_compile_opts() {
     RUSTFLAGS+=("-C target-cpu=${CPU}")
     CMAKE_FLAGS+=("-DCMAKE_C_FLAGS='${C_FLAGS[*]}'")
     CMAKE_FLAGS+=("-DCMAKE_CXX_FLAGS='${CXX_FLAGS[*]}'")
+    MESON_FLAGS+=("-Dc_args=${C_FLAGS[*]}" "-Dcpp_args=${CPP_FLAGS[*]}")
     dump_arr CONFIGURE_FLAGS
     dump_arr C_FLAGS
     dump_arr RUSTFLAGS
     dump_arr CMAKE_FLAGS
+    dump_arr MESON_FLAGS
     dump_arr PKG_CFG_FLAGS
 
     # extra ffmpeg flags
@@ -128,13 +141,32 @@ set_compile_opts() {
             '--nm=x86_64-w64-mingw32-gcc-nm'
         )
     fi
+    FFMPEG_EXTRA_FLAGS+=(
+        "--extra-cflags=\"${C_FLAGS[*]}\""
+        "--extra-cxxflags=\"${CXX_FLAGS[*]}\""
+        "--extra-ldflags=\"${LDFLAGS[*]}\""
+    )
     dump_arr FFMPEG_EXTRA_FLAGS
 
     # shellcheck disable=SC2178
     RUSTFLAGS="${RUSTFLAGS[*]}"
+
+    # make sure RUSTUP_HOME and CARGO_HOME are defined
+    if [[ "${RUSTUP_HOME}" == '' ]]; then
+        RUSTUP_HOME="${HOME}/.rustup"
+        test -d "${RUSTUP_HOME}" || echo_exit "RUSTUP_HOME does not exist"
+    fi
+    if [[ "${CARGO_HOME}" == '' ]]; then
+        CARGO_HOME="${HOME}/.rustup"
+        test -d "${CARGO_HOME}" || echo_exit "CARGO_HOME does not exist"
+    fi
+    export RUSTUP_HOME CARGO_HOME
+    unset SUDO_CARGO
+    if [[ "${SUDO}" != '' ]]; then
+        export SUDO_CARGO="${SUDO} --preserve-env=PATH,RUSTUP_HOME,CARGO_HOME"
+    fi
     echo
 }
-# set_compile_opts || return 1
 
 get_json_conf() {
     local build="${1}"
@@ -235,20 +267,19 @@ FB_FUNC_NAMES+=('build')
 # shellcheck disable=SC2034
 FB_FUNC_DESCS['build']='build ffmpeg with desired configuration'
 build() {
-    # set_compile_opts || return 1
-
     test -d "${DL_DIR}" || mkdir -p "${DL_DIR}"
     test -d "${CCACHE_DIR}" || mkdir -p "${CCACHE_DIR}"
     test -d "${BUILD_DIR}" || mkdir -p "${BUILD_DIR}"
-    ${SUDO} mkdir -p "${PREFIX}"
+    test -d "${PREFIX}/bin/" || mkdir -p "${PREFIX}/bin/"
 
+    unset SUDO
     testfile="${PREFIX}/ffmpeg-build-testfile"
     if ! touch "${testfile}" 2> /dev/null; then
         # we cannot modify the install prefix
-        # so we need to use sudo        
+        # so we need to use sudo
-        unset SUDO
         test "$(id -u)" -eq 0 || SUDO=sudo
         export SUDO
+        ${SUDO} mkdir -p "${PREFIX}/bin/"
     fi
     test -f "${testfile}" && ${SUDO} rm "${testfile}"
 
@@ -261,43 +292,73 @@ build() {
 }
 
 build_hdr10plus_tool() {
-    ccache cargo build --release
+    ccache cargo build --release || return 1
-    test -d "${PREFIX}/bin/" || ${SUDO} mkdir "${PREFIX}/bin/"
     ${SUDO} cp target/release/hdr10plus_tool "${PREFIX}/bin/" || return 1
 
     # build libhdr10plus
     cd hdr10plus || return 1
-    ccache cargo cbuild --release
+    ccache cargo cbuild --release || return 1
-    ${SUDO} bash -lc "cargo cinstall --prefix=${PREFIX} --release" || return 1
+    ${SUDO_CARGO} bash -lc "cargo cinstall --prefix=${PREFIX} --release" || return 1
 }
 
 build_dovi_tool() {
-    ccache cargo build --release
+    ccache cargo build --release || return 1
-    test -d "${PREFIX}/bin/" || ${SUDO} mkdir "${PREFIX}/bin/"
     ${SUDO} cp target/release/dovi_tool "${PREFIX}/bin/" || return 1
 
     # build libdovi
     cd dolby_vision || return 1
-    ccache cargo cbuild --release
+    ccache cargo cbuild --release || return 1
-    ${SUDO} bash -lc "cargo cinstall --prefix=${PREFIX} --release" || return 1
+    ${SUDO_CARGO} bash -lc "cargo cinstall --prefix=${PREFIX} --release" || return 1
+}
+
+build_libsvtav1() {
+    cmake \
+        "${CMAKE_FLAGS[@]}" \
+        -DSVT_AV1_LTO="${LTO_SWITCH}" \
+        -DENABLE_AVX512=ON \
+        -DBUILD_TESTING=OFF \
+        -DCOVERAGE=OFF || return 1
+    ccache make -j"${JOBS}" || return 1
+    ${SUDO} make -j"${JOBS}" install || return 1
 }
 
 build_libsvtav1_psy() {
     cmake \
         "${CMAKE_FLAGS[@]}" \
         -DSVT_AV1_LTO="${LTO_SWITCH}" \
-        -DENABLE_AVX512=ON \
         -DBUILD_TESTING=OFF \
+        -DENABLE_AVX512=ON \
         -DCOVERAGE=OFF \
         -DLIBDOVI_FOUND=1 \
-        -DLIBHDR10PLUS_RS_FOUND=1 || return 1
+        -DLIBHDR10PLUS_RS_FOUND=1 \
+        -DLIBHDR10PLUS_RS_LIBRARY="${MACHINE_LIB}/libhdr10plus-rs.${LIB_SUFF}" \
+        -DLIBDOVI_LIBRARY="${MACHINE_LIB}/libdovi.${LIB_SUFF}" || return 1
+    ccache make -j"${JOBS}" || return 1
+    ${SUDO} make -j"${JOBS}" install || return 1
+}
+
+build_librav1e() {
+    ccache cargo build --release || return 1
+    ${SUDO} cp target/release/rav1e "${PREFIX}/bin/" || return 1
+
+    ccache cargo cbuild --release || return 1
+    ${SUDO_CARGO} bash -lc "cargo cinstall --prefix=${PREFIX} --release" || return 1
+}
+
+build_libaom() {
+    cmake \
+        "${CMAKE_FLAGS[@]}" \
+        -B build.user \
+        -DENABLE_TESTS=OFF || return 1
+    cd build.user || return 1
     ccache make -j"${JOBS}" || return 1
     ${SUDO} make -j"${JOBS}" install || return 1
 }
 
 build_libopus() {
     ./configure \
-        "${CONFIGURE_FLAGS[@]}" || return 1
+        "${CONFIGURE_FLAGS[@]}" \
+        --disable-doc || return 1
     ccache make -j"${JOBS}" || return 1
     ${SUDO} make -j"${JOBS}" install || return 1
     return 0
@@ -306,14 +367,25 @@ build_libopus() {
 build_libdav1d() {
     meson \
         setup . build.user \
-        "${MESON_FLAGS[@]}" \
+        "${MESON_FLAGS[@]}" || return 1
-        -Dc_args="${C_FLAGS}" \
-        -Dcpp_args="${CPP_FLAGS}" \
-        -Db_lto="${LTO_BOOL}" || return 1
     ccache ninja -vC build.user || return 1
     ${SUDO} ninja -vC build.user install || return 1
 }
 
+build_libvmaf() {
+    cd libvmaf || return 1
+    python3 -m virtualenv .venv
+    (
+        source .venv/bin/activate
+        meson \
+            setup . build.user \
+            "${MESON_FLAGS[@]}" \
+            -Denable_float=true || exit 1
+        ccache ninja -vC build.user || exit 1
+        ${SUDO} ninja -vC build.user install || exit 1
+    ) || return 1
+}
+
 build_ffmpeg() {
     for enable in "${FFMPEG_ENABLES[@]}"; do
         test "${enable}" == 'libsvtav1_psy' && enable='libsvtav1'
@@ -324,13 +396,9 @@ build_ffmpeg() {
         "${FFMPEG_EXTRA_FLAGS[@]}" \
         --pkg-config='pkg-config' \
         --pkg-config-flags="${PKG_CFG_FLAGS}" \
-        --disable-debug --enable-nonfree \
-        --enable-gpl --enable-version3 \
         --cpu="${CPU}" --arch="${ARCH}" \
-        --extra-cflags="${C_FLAGS[*]}" \
+        --enable-gpl --enable-version3 \
-        --extra-cxxflags="${CXX_FLAGS[*]}" \
+        --enable-nonfree \
-        --extra-ldflags="${LDFLAGS}" \
-        --disable-doc \
         --disable-htmlpages \
         --disable-podpages \
         --disable-txtpages \

scripts/common.sh

@@ -19,19 +19,46 @@ void() { echo "$@" >/dev/null ; }
 
 echo_if_fail() {
     local cmd=("$@")
-    local out="/tmp/.stdout-${RANDOM}"
+    local out="${TMP_DIR}/.stdout-${RANDOM}"
-    local err="/tmp/.stderr-${RANDOM}"
+    local err="${TMP_DIR}/.stderr-${RANDOM}"
-    "${cmd[@]}" > ${out} 2> ${err}
+    test -d "${TMP_DIR}" || mkdir -p "${TMP_DIR}"
+
+    # set trace to the cmdEvalTrace and open file descriptor
+    local cmdEvalTrace="${TMP_DIR}/.cmdEvalTrace-${RANDOM}"
+    exec 5> "${cmdEvalTrace}"
+    export BASH_XTRACEFD=5
+
+    set -x
+    "${cmd[@]}" >"${out}" 2>"${err}"
     local retval=$?
+
+    # unset and close file descriptor
+    set +x
+    exec 5>&-
+    
+    # parse out relevant part of the trace
+    local cmdEvalLines=()
+    cmd=()
+    while IFS= read -r line; do
+        cmdEvalLines+=("${line}")
+    done < "${cmdEvalTrace}"
+    local cmdEvalLineNum=${#cmdEvalLines[@]}
+    for ((i=1; i < cmdEvalLineNum-2; i++)); do
+        local trimmedCmd="${cmdEvalLines[${i}]}"
+        trimmedCmd="${trimmedCmd/+ /}"
+        cmd+=("${trimmedCmd}")
+    done
+
     if ! test ${retval} -eq 0; then
         echo
-        echo_fail "command [" "${cmd[@]}" "] failed"
+        echo_fail "command failed:" 
+        printf "%s\n" "${cmd[@]}"
         echo_warn "command output:"
-        tail -n 10 ${out}
+        tail -n 10 "${out}"
-        tail -n 10 ${err}
+        tail -n 10 "${err}"
         echo
     fi
-    rm ${out} ${err}
+    rm "${out}" "${err}" "${cmdEvalTrace}"
     return ${retval}
 }