tsort deps to minimize rebuilds

lib/0-utils.sh

@@ -37,24 +37,55 @@ echo_exit() {
 }
 void() { echo "$@" >/dev/null; }
 
-echo_if_fail() {
+trace_cmd_to_file() {
     local cmd=("$@")
-    local logName="${LOGNAME:-${RANDOM}}-"
+    test -n "${OUT}" && test -n "${ERR}" && test -n "${TRACE}" || return 1
-    local out="${TMP_DIR}/${logName}stdout"
+    local out="${OUT}"
-    local err="${TMP_DIR}/${logName}stderr"
+    local err="${ERR}"
+    local trace="${TRACE}"
+    local fd=6
+
+    # get current trace status
+    local set
+    if [[ $- == *x* ]]; then
+        set='-x'
+    else
+        set='+x'
+    fi
+
+    # stop tracing before new trace file
+    set +x
 
     # set trace to the cmdEvalTrace and open file descriptor
-    local cmdEvalTrace="${TMP_DIR}/${logName}cmdEvalTrace"
+    eval "exec ${fd}>\"${trace}\""
-    exec 5>"${cmdEvalTrace}"
+    export BASH_XTRACEFD=${fd}
-    export BASH_XTRACEFD=5
 
     set -x
     "${cmd[@]}" >"${out}" 2>"${err}"
     local retval=$?
 
     # unset and close file descriptor
+    # unset BASH_XTRACEFD first
     set +x
-    exec 5>&-
+    unset BASH_XTRACEFD
+    eval "exec ${fd}>&-"
+
+    # reset previous state
+    set ${set}
+
+    return ${retval}
+}
+
+echo_if_fail() {
+    local cmd=("$@")
+    local logName="${LOGNAME:-${RANDOM}}-"
+    local out="${TMP_DIR}/${logName}stdout"
+    local err="${TMP_DIR}/${logName}stderr"
+    local trace="${TMP_DIR}/${logName}trace"
+
+    OUT="${out}" ERR="${err}" TRACE="${trace}" trace_cmd_to_file \
+        "${cmd[@]}"
+    local retval=$?
 
     # parse out relevant part of the trace
     local cmdEvalLines=()
@@ -63,7 +94,7 @@ echo_if_fail() {
         test "${line}" == 'set +x' && continue
         test "${line}" == '' && continue
         cmdEvalLines+=("${line}")
-    done <"${cmdEvalTrace}"
+    done <"${trace}"
 
     if ! test ${retval} -eq 0; then
         echo
@@ -76,7 +107,7 @@ echo_if_fail() {
         echo
     fi
     if [[ -z ${LOGNAME} ]]; then
-        rm "${out}" "${err}" "${cmdEvalTrace}"
+        rm "${out}" "${err}" "${trace}"
     fi
     return ${retval}
 }
@@ -333,19 +364,14 @@ bash_sort() {
 }
 
 _start_spinner() {
-    local spinChars=(
+    local spinChars=("⠋" "⠙" "⠹" "⠸" "⠼" "⠴" "⠦" "⠧" "⠇" "⠏")
-        "-"
-        '\'
-        "|"
-        "/"
-    )
 
     sleep 1
 
     while true; do
         for ((ind = 0; ind < "${#spinChars[@]}"; ind++)); do
             echo -ne "${spinChars[${ind}]}" '\b\b'
-            sleep .25
+            sleep .125
         done
     done
 }
@@ -412,11 +438,29 @@ ensure_dir() {
     done
 }
 
+get_date() {
+    printf '%(%Y-%m-%d)T\n' -1
+}
+
 get_remote_head() {
     local url="$1"
-    local remoteHEAD=''
+    # ${build} is magically populated from get_build_conf
+    local remoteCommitFile="${TMP_DIR}/${build}-remote-head.txt"
+    local date="$(get_date)"
+    # want to cache the remote head for faster retrieval
+    # and only get it once per day
+    local remoteHEAD
+    if [[ -f ${remoteCommitFile} ]]; then
+        IFS=' ' read -r prevDate prevBuild prevCommit <"${remoteCommitFile}"
+    fi
+
+    if [[ "${date}:${build}" == "${prevDate}:${prevBuild}" ]]; then
+        remoteHEAD="${prevCommit}"
+    else
         IFS=$' \t' read -r remoteHEAD _ <<< \
             "$(git ls-remote "${url}" HEAD)"
+        echo "${date} ${build} ${remoteHEAD}" >"${remoteCommitFile}"
+    fi
     echo "${remoteHEAD}"
 }
 
@@ -436,3 +480,51 @@ print_padded() {
         echo -n ' '
     done
 }
+
+benchmark_command() {
+    local cmd=("$@")
+    local out='/dev/null'
+    local err='/dev/null'
+    local trace="${TMP_DIR}/benchmark-trace.txt"
+
+    # note this marker around the awk parsing
+    local traceMarker='TRACEOUTPUT ; '
+
+    echo_warn "tracing ${cmd[*]}"
+    PS4='$EPOCHREALTIME TRACEOUTPUT ; ' OUT="${out}" ERR="${err}" TRACE="${trace}" trace_cmd_to_file \
+        "${cmd[@]}" || return 1
+    echo_pass "done tracing ${cmd[*]}"
+
+    awk '
+    BEGIN { 
+        # Use a regex for the marker to handle leading/trailing whitespace cleanly
+        marker = "TRACEOUTPUT ; " 
+    }
+    
+    # Match lines starting with a timestamp followed by the marker
+    $0 ~ /^[0-9.]+[ ]+TRACEOUTPUT ; / {
+        # 1. Calculate duration for the PREVIOUS command now that we have a new timestamp
+        if (prev_time != "") {
+            duration = $1 - prev_time
+            # Output for the system sort: [duration] | [command]
+            printf "%012.6f | %s\n", duration, prev_cmd
+        }
+        
+        # 2. Start tracking the NEW command
+        prev_time = $1
+        
+        # Extract everything after "TRACEOUTPUT ; "
+        # match() finds the position, substr() grabs the rest
+        match($0, /TRACEOUTPUT ; /)
+        prev_cmd = substr($0, RSTART + RLENGTH)
+        next
+    }
+
+    # If the line does not match the timestamp pattern, it is a newline in a variable
+    {
+        if (prev_time != "") {
+            prev_cmd = prev_cmd "\n" $0
+        }
+    }
+    ' "${trace}" | sort -rn | head -n 20
+}

lib/build.sh

@@ -368,7 +368,7 @@ glad              2.0.8        tar.gz    https://github.com/Dav1dde/glad/archive
 libx265           4.1          tar.gz    https://bitbucket.org/multicoreware/x265_git/downloads/x265_${ver}.${ext} libnuma
 libnuma           2.0.19       tar.gz    https://github.com/numactl/numactl/archive/refs/tags/v${ver}.${ext}
 
-libass            0.17.4       tar.xz    https://github.com/libass/libass/releases/download/${ver}/libass-${ver}.${ext} libfreetype,libharfbuzz,libfribidi,libunibreak,libxml2,xz
+libass            0.17.4       tar.xz    https://github.com/libass/libass/releases/download/${ver}/libass-${ver}.${ext} libfontconfig,libfreetype,libharfbuzz,libfribidi,libunibreak,libxml2,xz
 libfontconfig     2.17.1       tar.xz    https://gitlab.freedesktop.org/api/v4/projects/890/packages/generic/fontconfig/${ver}/fontconfig-${ver}.${ext} libharfbuzz,expat,brotli
 libfreetype       2.14.1       tar.xz    https://downloads.sourceforge.net/freetype/freetype-${ver}.${ext} bzip,libpng,zlib,brotli
 libharfbuzz       12.3.0       tar.xz    https://github.com/harfbuzz/harfbuzz/releases/download/${ver}/harfbuzz-${ver}.${ext} libfreetype
@@ -406,7 +406,7 @@ expat             2.7.3        tar.xz    https://github.com/libexpat/libexpat/re
     fi
 
     # special arg to print BUILDS_CONF but formatted with spaces
-    if [[ ${getBuild} == 'format-builds-conf' ]]; then
+    if [[ ${getBuild} == 'formatted' ]]; then
         echo "local BUILDS_CONF='"
         while read -r line; do
             IFS=$' \t' read -r build ver ext url deps <<<"${line}"
@@ -428,8 +428,15 @@ expat             2.7.3        tar.xz    https://github.com/libexpat/libexpat/re
     # url uses ver and extension
     eval "url=\"$url\""
     # set dependencies array
+    # ffmpeg dependencies are everything enabled
+    if [[ ${build} == 'ffmpeg' ]]; then
+        # shellcheck disable=SC2206
+        # shellcheck disable=SC2153
+        deps=(${ENABLE})
+    else
         # shellcheck disable=SC2206
         deps=(${deps//,/ })
+    fi
     # set version based off of remote head
     # and set extracted directory
     if [[ ${ext} == 'git' ]]; then
@@ -504,7 +511,7 @@ download_release() {
             cd "${download}" || exit 1
             local localHEAD remoteHEAD
             localHEAD="$(git rev-parse HEAD)"
-            remoteHEAD="$(get_remote_head "$(git config --get remote.origin.url)")"
+            remoteHEAD="$(get_remote_head "${url}")"
             if [[ ${localHEAD} != "${remoteHEAD}" ]]; then
                 git stash
                 git pull --ff-only
@@ -522,6 +529,29 @@ download_release() {
     fi
 }
 
+# given a build, topologically sort
+# a build and its dependencies
+# to minimize rebuilds
+_generate_build_order() (
+    local build="$1"
+    local depMap=("${build} ${build}")
+    get_build_conf "${build}" || return 1
+    for dep in "${deps[@]}"; do
+        _generate_build_order "${dep}" || return 1
+        depMap+=("${dep} ${build}")
+    done
+    printf '%s\n' "${depMap[@]}"
+)
+generate_build_order() {
+    local build="$1"
+    local order
+    while read -r line; do
+        order+="${line} "
+    done <<<"$(_generate_build_order "${build}" | tsort)"
+    # remove self-inclusive build from order list
+    echo "${order// ${build}/}"
+}
+
 FB_FUNC_NAMES+=('do_build')
 # shellcheck disable=SC2034
 FB_FUNC_DESCS['do_build']='build a specific project'
@@ -530,14 +560,22 @@ FB_FUNC_COMPLETION['do_build']="$(get_build_conf supported)"
 do_build() {
     local build="${1:-''}"
     get_build_conf "${build}" || return 1
-    download_release || return 1
 
     set_compile_opts || return 1
-    for dep in "${deps[@]}"; do
+
-        do_build "${dep}" || return 1
+    # set BUILD_ORDER only if unset
+    # this recursive function needs to be able
+    # to control its callstack to minimize re-calls
+    if [[ ${BUILD_ORDER} == '' ]]; then
+        BUILD_ORDER="$(generate_build_order "${build}")" || return 1
+        for b in ${BUILD_ORDER}; do
+            test "${b}" == "${build}" && continue
+            do_build "${b}" || return 1
         done
+        unset BUILD_ORDER
+    fi
+
     get_build_conf "${build}" || return 1
-    download_release || return 1
 
     # save the metadata for a build to skip re-building identical builds
     local oldMetadataFile="${TMP_DIR}/${build}-old-metadata"
@@ -557,6 +595,10 @@ do_build() {
         echo "ver: ${ver}"
         echo "url: ${url}"
         echo "LOCAL_PREFIX: ${LOCAL_PREFIX}"
+        for patch in "${PATCHES_DIR}/${build}"/*.patch; do
+            test -f "${patch}" || continue
+            echo "patch:${patch}" >>"${newMetadataFile}"
+        done
         COLOR=OFF SHOW_SINGLE=true dump_arr "${BUILD_ENV_NAMES[@]}"
     } >"${newMetadataFile}"
 
@@ -567,20 +609,20 @@ do_build() {
         type add_project_versioning_to_ffmpeg >>"${newMetadataFile}"
     fi
 
-    # prepare build
-    pushd "${extractedDir}" >/dev/null || return 1
-    # check for any patches
-    for patch in "${PATCHES_DIR}/${build}"/*.patch; do
-        test -f "${patch}" || continue
-        echo_if_fail patch -p1 -i "${patch}" || return 1
-        echo "patch:${patch}" >>"${newMetadataFile}"
-    done
-
     # rebuild if new metadata is different
     local newMetadata="$(<"${newMetadataFile}")"
     local oldMetadata=''
     test -f "${oldMetadataFile}" && oldMetadata="$(<"${oldMetadataFile}")"
     if [[ ${oldMetadata} != "${newMetadata}" || -n ${REQUIRES_REBUILD} ]]; then
+        # prepare build
+        download_release || return 1
+        pushd "${extractedDir}" >/dev/null || return 1
+        # check for any patches
+        for patch in "${PATCHES_DIR}/${build}"/*.patch; do
+            test -f "${patch}" || continue
+            echo_if_fail patch -p1 -i "${patch}" || return 1
+        done
+
         echo_info -n "building ${build} "
         # build in background
         local timeBefore=${EPOCHSECONDS}
@@ -603,7 +645,6 @@ do_build() {
         # indicate that build chain will require rebuild
         REQUIRES_REBUILD=1
     else
-        popd >/dev/null || return 1
         echo_info "re-using identical previous build for ${build}"
     fi
 }
@@ -622,11 +663,6 @@ build() {
 
     set_compile_opts || return 1
 
-    for build in ${ENABLE}; do
-        do_build "${build}" || return 1
-        # reset whether build chain requires a rebuild
-        unset REQUIRES_REBUILD
-    done
     do_build ffmpeg || return 1
 
     # skip packaging on PGO generate run

main.sh

@@ -33,11 +33,6 @@ for dir in "${IGN_DIRS[@]}"; do
 done
 unset IGN_DIRS
 
-# function names, descriptions, completions
-unset FB_FUNC_NAMES FB_FUNC_DESCS FB_FUNC_COMPLETION
-FB_FUNC_NAMES=()
-declare -A FB_FUNC_DESCS FB_FUNC_COMPLETION
-
 # can't have recursive generation
 FB_RUNNING_AS_SCRIPT=${FB_RUNNING_AS_SCRIPT:-0}
 
@@ -46,10 +41,34 @@ FB_COMPILE_OPTS_SET=0
 
 SCRIPT_DIR="${REPO_DIR}/scripts"
 ENTRY_SCRIPT="${SCRIPT_DIR}/entry.sh"
-src_scripts() {
-    local SCRIPT_DIR="${REPO_DIR}/scripts"
 
-    if [[ $FB_RUNNING_AS_SCRIPT -eq 0 ]]; then
+src_scripts() {
+    # function names, descriptions, completions
+    unset FB_FUNC_NAMES FB_FUNC_DESCS FB_FUNC_COMPLETION
+    FB_FUNC_NAMES=()
+    declare -gA FB_FUNC_DESCS FB_FUNC_COMPLETION
+    for script in "${REPO_DIR}/lib/"*.sh; do
+        # shellcheck disable=SC1090
+        source "${script}"
+    done
+    for funcName in "${FB_FUNC_NAMES[@]}"; do
+        (cd "${SCRIPT_DIR}" && ln -sf "entry.sh" "${funcName}.sh")
+    done
+}
+src_scripts || return 1
+
+FB_FUNC_NAMES+=('print_cmds')
+FB_FUNC_DESCS['print_cmds']='print usable commands'
+print_cmds() {
+    echo -e "~~~ Usable Commands ~~~\n"
+    for funcName in "${FB_FUNC_NAMES[@]}"; do
+        color="${CYAN}" word="${funcName}:" echo_wrapper "\n\t${FB_FUNC_DESCS[${funcName}]}"
+    done
+    echo -e "\n"
+}
+
+gen_links() {
+    if [[ ${FB_RUNNING_AS_SCRIPT} -eq 0 ]]; then
         rm "${SCRIPT_DIR}"/*.sh
         # shellcheck disable=SC2016
         echo '#!/usr/bin/env bash
@@ -62,26 +81,12 @@ cmd="${scr_name//.sh/}"
 if [[ $DEBUG == 1 ]]; then set -x; fi
 $cmd "$@"' >"${ENTRY_SCRIPT}"
         chmod +x "${ENTRY_SCRIPT}"
-    fi
-
-    for script in "${REPO_DIR}/lib/"*.sh; do
-        # shellcheck disable=SC1090
-        source "${script}"
-    done
-}
-
-FB_FUNC_NAMES+=('print_cmds')
-FB_FUNC_DESCS['print_cmds']='print usable commands'
-print_cmds() {
-    echo -e "~~~ Usable Commands ~~~\n"
         for funcName in "${FB_FUNC_NAMES[@]}"; do
-        color="${CYAN}" word="${funcName}:" echo_wrapper "\n\t${FB_FUNC_DESCS[${funcName}]}"
+            (cd "${SCRIPT_DIR}" && ln -sf "entry.sh" "${funcName}.sh")
-        if [[ $FB_RUNNING_AS_SCRIPT -eq 0 ]]; then
-            (cd "$SCRIPT_DIR" && ln -sf entry.sh "${funcName}.sh")
-        fi
         done
-    echo -e "\n"
+    fi
 }
+gen_links || return 1
 
 set_completions() {
     for funcName in "${FB_FUNC_NAMES[@]}"; do
@@ -89,7 +94,6 @@ set_completions() {
     done
 }
 
-src_scripts || return 1
 determine_pkg_mgr || return 1
 check_compile_opts_override || return 1