diff --git a/.gitignore b/.gitignore index bb9c2ad..702b5ce 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ benchmark repos out .last_opts +.good_commits diff --git a/scripts/build.sh b/scripts/build.sh index c97b9ce..5757932 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -96,22 +96,6 @@ while getopts "$OPTS" flag; do esac done -GIT_DEPTH='5' -update_git() { - # don't stash this actual repo - CURRENT_REPO="$(basename "$(git rev-parse --show-toplevel)")" - if [[ "$CURRENT_REPO" == "ffmpeg-av1-builder" ]]; then - exit 1 - fi - git config pull.rebase false - git stash && git stash drop - PRE_COMMIT="$(git rev-parse HEAD)" - git pull || exit 1 - POST_COMMIT="$(git rev-parse HEAD)" - test "$PRE_COMMIT" != "$POST_COMMIT" && return 1 - return 0 -} - # set default optimization level if [[ -z $OPT_LVL ]]; then OPT_LVL=3 @@ -144,6 +128,34 @@ mkdir "$REPOS_DIR" # save options use echo "$@" > "$BASE_DIR/.last_opts" +# save which commits were successful +GOOD_COMMIT_BUILDS="$BASE_DIR/.good_commits" +good_commit_output() { + echo "$(git remote -v | head -n1 | awk '{print $2}' | sed 's/.*\///' | sed 's/\.git//'):$(git rev-parse HEAD)" +} +set_commit_status() { + grep -q "$(good_commit_output)" "$GOOD_COMMIT_BUILDS" || good_commit_output >> "$GOOD_COMMIT_BUILDS" +} + + +GIT_DEPTH='5' +check_for_rebuild() { + # don't stash this actual repo + CURRENT_REPO="$(basename "$(git rev-parse --show-toplevel)")" + if [[ "$CURRENT_REPO" == "ffmpeg-av1-builder" ]]; then + exit 1 + fi + git config pull.rebase false + git stash && git stash drop + PRE_COMMIT="$(git rev-parse HEAD)" + git pull || exit 1 + POST_COMMIT="$(git rev-parse HEAD)" + test "$PRE_COMMIT" != "$POST_COMMIT" && return 1 + # check if project built successfully + grep -q "$(good_commit_output)" "$GOOD_COMMIT_BUILDS" || return 1 + return 0 +} + # prefix to install PREFIX='/usr/local' @@ -209,10 +221,13 @@ if [[ "$(uname -r)" =~ "WSL" ]] ; then sudo hwclock -s fi +# clone ffmpeg +git clone --depth "$GIT_DEPTH" https://github.com/FFmpeg/FFmpeg "$FFMPEG_DIR" + build_mpp() { # build mpp - git clone --depth "$GIT_DEPTH" -b jellyfin-mpp https://github.com/nyanmisaka/mpp.git "$RKMPP_DIR" || { update_git && return 0 ; } - cd "$RKMPP_DIR/" || return 1 + git clone --depth "$GIT_DEPTH" -b jellyfin-mpp https://github.com/nyanmisaka/mpp.git "$RKMPP_DIR" || \ + { cd "$RKMPP_DIR/" && check_for_rebuild && return 0 ; } rm -rf mpp_build.user mkdir mpp_build.user cd mpp_build.user || return 1 @@ -230,8 +245,8 @@ build_mpp() { build_rkga() { # build rga - git clone --depth "$GIT_DEPTH" -b jellyfin-rga https://github.com/nyanmisaka/rk-mirrors.git "$RKRGA_DIR" || { update_git && return 0 ; } - cd "$RKRGA_DIR" || return 1 + git clone --depth "$GIT_DEPTH" -b jellyfin-rga https://github.com/nyanmisaka/rk-mirrors.git "$RKRGA_DIR" || \ + { cd "$RKRGA_DIR" && check_for_rebuild && return 0 ; } rm -rf rga_build.user mkdir rga_build.user meson setup . rga_build.user \ @@ -263,8 +278,8 @@ fi build_dovi() { # build dovi_tool - git clone --depth "$GIT_DEPTH" https://github.com/quietvoid/dovi_tool "$DOVI_DIR" || { update_git && return 0 ; } - cd "$DOVI_DIR/" || return 1 + git clone --depth "$GIT_DEPTH" https://github.com/quietvoid/dovi_tool "$DOVI_DIR" || \ + { cd "$DOVI_DIR/" && check_for_rebuild && return 0 ; } rm -rf ffmpeg_build.user && mkdir ffmpeg_build.user || return 1 source "$HOME/.cargo/env" # for good measure cargo clean @@ -275,13 +290,13 @@ build_dovi() { cd dolby_vision || return 1 RUSTFLAGS="-C target-cpu=native" ccache cargo cbuild --release sudo -E bash -lc "cargo cinstall --prefix=${PREFIX} --release" || return 1 - + set_commit_status } build_hdr10plus() { # build hdr10plus_tool - git clone --depth "$GIT_DEPTH" https://github.com/quietvoid/hdr10plus_tool "$HDR10_DIR" || { update_git && return 0 ; } - cd "$HDR10_DIR/" || return 1 + git clone --depth "$GIT_DEPTH" https://github.com/quietvoid/hdr10plus_tool "$HDR10_DIR" || \ + { cd "$HDR10_DIR/" && check_for_rebuild && return 0 ; } rm -rf ffmpeg_build.user && mkdir ffmpeg_build.user || return 1 source "$HOME/.cargo/env" # for good measure cargo clean @@ -292,14 +307,28 @@ build_hdr10plus() { cd hdr10plus || return 1 RUSTFLAGS="-C target-cpu=native" ccache cargo cbuild --release sudo -E bash -lc "cargo cinstall --prefix=${PREFIX} --release" || return 1 + set_commit_status } build_svt_av1_psy() { # build svt-avt-psy - # clear svt because of unrelated histories error - rm -rf "$SVT_PSY_DIR" - git clone --depth "$GIT_DEPTH" https://github.com/gianni-rosato/svt-av1-psy "$SVT_PSY_DIR" - cd "$SVT_PSY_DIR/" || return 1 + # svt-av1-psy cannot be cleanly updated + # due to histories error + local GIT_REPO_URL='https://github.com/gianni-rosato/svt-av1-psy' + git clone --depth "$GIT_DEPTH" "$GIT_REPO_URL" "$SVT_PSY_DIR" || \ + { + cd "$SVT_PSY_DIR/" || return 1 + REMOTE_HEAD_SHA="$(git ls-remote "$GIT_REPO_URL" HEAD | awk '{ print $1 }')" + CURR_HEAD_SHA="$(git rev-parse HEAD)" + if [[ "$REMOTE_HEAD_SHA" != "$CURR_HEAD_SHA" ]]; then + # wipe and start over + rm -rf "$SVT_PSY_DIR" + git clone --depth "$GIT_DEPTH" "$GIT_REPO_URL" "$SVT_PSY_DIR" + else + return 0; + fi + } + # build in tmp because some artifacts # are installed with sudo and then cannot # be removed later @@ -323,12 +352,13 @@ build_svt_av1_psy() { "$SVT_PSY_DIR" || return 1 ccache make -j"${THREADS}" || return 1 sudo make install + set_commit_status } build_svt_av1() { # build svt-av1 - git clone --depth "$GIT_DEPTH" https://gitlab.com/AOMediaCodec/SVT-AV1.git "$SVT_DIR" || { update_git && return 0 ; } - cd "$SVT_DIR/" || return 1 + git clone --depth "$GIT_DEPTH" https://gitlab.com/AOMediaCodec/SVT-AV1.git "$SVT_DIR" || \ + { cd "$SVT_DIR/" && check_for_rebuild && return 0 ; } rm -rf build_svt.user mkdir build_svt.user cd build_svt.user || return 1 @@ -344,6 +374,7 @@ build_svt_av1() { -DCMAKE_CXX_FLAGS="-O${OPT_LVL} ${COMP_FLAGS}" || return 1 ccache make -j"${THREADS}" || return 1 sudo make install || return 1 + set_commit_status } if [[ "$BUILD_PSY" == "Y" ]]; @@ -357,20 +388,20 @@ fi build_rav1e() { # build rav1e - git clone --depth "$GIT_DEPTH" https://github.com/xiph/rav1e "$RAV1E_DIR" || { update_git && return 0 ; } - cd "$RAV1E_DIR/" || return 1 + git clone --depth "$GIT_DEPTH" https://github.com/xiph/rav1e "$RAV1E_DIR" || \ + { cd "$RAV1E_DIR/" && check_for_rebuild && return 0 ; } rm -rf ffmpeg_build.user && mkdir ffmpeg_build.user || return 1 source "$HOME/.cargo/env" # for good measure cargo clean RUSTFLAGS="-C target-cpu=native" ccache cargo cbuild --release sudo -E bash -lc "cargo cinstall --prefix=${PREFIX} --release" || return 1 - + set_commit_status } build_aom_av1() { # build aom - git clone --depth "$GIT_DEPTH" https://aomedia.googlesource.com/aom "$AOM_DIR" || { update_git && return 0 ; } - cd "$AOM_DIR/" || return 1 + git clone --depth "$GIT_DEPTH" https://aomedia.googlesource.com/aom "$AOM_DIR" || \ + { cd "$AOM_DIR/" && check_for_rebuild && return 0 ; } rm -rf build_aom.user mkdir build_aom.user cd build_aom.user || return 1 @@ -383,7 +414,7 @@ build_aom_av1() { -DCMAKE_CXX_FLAGS="${LTO_FLAG} -O${OPT_LVL} ${COMP_FLAGS}" || return 1 ccache make -j"${THREADS}" || return 1 sudo make install || return 1 - + set_commit_status } if [[ "$BUILD_ALL_AV1" == "Y" ]]; then @@ -393,8 +424,8 @@ fi build_vmaf() { # build libvmaf - git clone --depth "$GIT_DEPTH" https://github.com/Netflix/vmaf "$VMAF_DIR" || { update_git && return 0 ; } - cd "$VMAF_DIR/libvmaf" || exit + git clone --depth "$GIT_DEPTH" https://github.com/Netflix/vmaf "$VMAF_DIR" || \ + { cd "$VMAF_DIR/libvmaf" && check_for_rebuild && return 0 ; } python3 -m virtualenv .venv ( source .venv/bin/activate @@ -412,16 +443,17 @@ build_vmaf() { ccache ninja -vC build.user || exit sudo ninja -vC build.user install || exit ) + set_commit_status } if [[ "$BUILD_VMAF" == "Y" ]]; then - build_vmaf + build_vmaf || exit 1 fi build_dav1d() { # build dav1d - git clone --depth "$GIT_DEPTH" https://code.videolan.org/videolan/dav1d.git "$DAV1D_DIR" || { update_git && return 0 ; } - cd "$DAV1D_DIR" || return 1 + git clone --depth "$GIT_DEPTH" https://code.videolan.org/videolan/dav1d.git "$DAV1D_DIR" || \ + { cd "$DAV1D_DIR" && check_for_rebuild && return 0 ; } rm -rf build.user mkdir build.user meson setup . build.user \ @@ -433,12 +465,13 @@ build_dav1d() { -Dcpp_args="${COMP_FLAGS}" || return 1 ccache ninja -vC build.user || return 1 sudo ninja -vC build.user install || return 1 + set_commit_status } build_opus() { # build opus - git clone --depth "$GIT_DEPTH" https://github.com/xiph/opus.git "$OPUS_DIR" || { update_git && return 0 ; } - cd "$OPUS_DIR" || return 1 + git clone --depth "$GIT_DEPTH" https://github.com/xiph/opus.git "$OPUS_DIR" || \ + { cd "$OPUS_DIR" && check_for_rebuild && return 0 ; } ./autogen.sh || return 1 CFLAGS="-O${OPT_LVL} ${LTO_FLAG} ${COMP_FLAGS}" export CFLAGS @@ -447,6 +480,7 @@ build_opus() { ccache make -j"${THREADS}" || return 1 sudo make install || return 1 unset CFLAGS + set_commit_status } build_dav1d || exit 1 @@ -455,8 +489,8 @@ build_opus || exit 1 build_x264() { # build x264 - git clone --depth "$GIT_DEPTH" https://code.videolan.org/videolan/x264.git "$X264_DIR" || { update_git && return 0 ; } - cd "$X264_DIR" || return 1 + git clone --depth "$GIT_DEPTH" https://code.videolan.org/videolan/x264.git "$X264_DIR" || \ + { cd "$X264_DIR" && check_for_rebuild && return 0 ; } make clean ./configure --enable-static \ --enable-pic \ @@ -465,7 +499,7 @@ build_x264() { --extra-cflags="-O${OPT_LVL} ${COMP_FLAGS}" || return 1 ccache make -j"${THREADS}" || return 1 sudo make install || return 1 - + set_commit_status } build_x265() { @@ -473,12 +507,7 @@ build_x265() { git clone --depth "$GIT_DEPTH" https://bitbucket.org/multicoreware/x265_git.git "$X265_DIR" cd "$X265_DIR" || return 1 test -d ".no_git" && mv .no_git .git - test -d ".git" && git stash && git stash drop - test -d ".git" && git config pull.rebase false - PRE_COMMIT="$(git rev-parse HEAD)" - test -d ".git" && git pull - POST_COMMIT="$(git rev-parse HEAD)" - test "$PRE_COMMIT" == "$POST_COMMIT" && return 0 + test -d ".git" && check_for_rebuild && return 0 # x265 is dumb and only generates pkgconfig # if git is not there ("release") mv .git .no_git @@ -493,20 +522,20 @@ build_x265() { -DCMAKE_INSTALL_PREFIX="${PREFIX}" \ -DEXPORT_C_API=ON \ -DENABLE_SHARED=ON \ - -DCMAKE_C_FLAGS="${LTO_CONFIGURE} -O${OPT_LVL} ${COMP_FLAGS}" \ - -DCMAKE_CXX_FLAGS="${LTO_CONFIGURE} -O${OPT_LVL} ${COMP_FLAGS}" || return 1 + -DCMAKE_C_FLAGS="-O${OPT_LVL} ${COMP_FLAGS}" \ + -DCMAKE_CXX_FLAGS="-O${OPT_LVL} ${COMP_FLAGS}" || return 1 ccache make -j"${THREADS}" || return 1 sudo make install || return 1 cd "$X265_DIR" || return 1 # revert git mv .no_git .git - + set_commit_status } build_vpx() { # build vpx - git clone --depth "$GIT_DEPTH" https://chromium.googlesource.com/webm/libvpx.git "$VPX_DIR" || { update_git && return 0 ; } - cd "$VPX_DIR" || return 1 + git clone --depth "$GIT_DEPTH" https://chromium.googlesource.com/webm/libvpx.git "$VPX_DIR" || \ + { cd "$VPX_DIR" && check_for_rebuild && return 0 ; } if [[ "$ARCH" == "x86_64" ]]; then VP_COMP_FLAGS="${COMP_FLAGS}"; else @@ -526,7 +555,7 @@ build_vpx() { --enable-vp9-highbitdepth ccache make -j"${THREADS}" || return 1 sudo make install || return 1 - + set_commit_status } if [[ "$BUILD_OTHERS" == "Y" ]]; then @@ -536,7 +565,7 @@ if [[ "$BUILD_OTHERS" == "Y" ]]; then # # build gtest # git clone --depth "$GIT_DEPTH" https://github.com/google/googletest "$GTEST_DIR" # cd "$GTEST_DIR" || exit - # update_git + # check_for_rebuild # rm -rf build # mkdir build # cd build || exit @@ -553,8 +582,7 @@ if command -v ldconfig ; then fi # build ffmpeg -git clone --depth "$GIT_DEPTH" https://github.com/FFmpeg/FFmpeg "$FFMPEG_DIR" || update_git -cd "$FFMPEG_DIR/" || exit +cd "$FFMPEG_DIR/" && check_for_rebuild export PKG_CONFIG_PATH="${PREFIX}/lib/pkgconfig:$PKG_CONFIG_PATH" make clean ./configure --enable-libsvtav1 \ @@ -573,6 +601,7 @@ make clean --disable-txtpages || exit ccache make -j"${THREADS}" || exit sudo make install || exit +set_commit_status sudo cp ff*_g "${PREFIX}/bin/" # validate encoders