diff --git a/Jenkinsfile b/Jenkinsfile index 7cf41ad..fb0e1c0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -61,7 +61,7 @@ pipeline { agent { label "linux && ${ARCH}" } steps { withDockerCreds { - sh "${OPT_LTO} ./scripts/docker_run_image.sh ${DISTRO}" + sh "${OPT_LTO} ./scripts/build_with_docker.sh ${DISTRO}" } } } diff --git a/README.md b/README.md index 16a6e7a..3602e16 100644 --- a/README.md +++ b/README.md @@ -1 +1,45 @@ -# ffmpeg-builder \ No newline at end of file + +A collection of scripts for building ffmpeg and encoding content with the built ffmpeg + +Tested on: +- linux x86_64/aarch64 on: + - ubuntu + - fedora + - debian + - archlinux +- darwin aarch64 + +```bash +~~~ Usable Commands ~~~ + +print_cmds: + print usable commands +do_build: + build a specific project +build: + build ffmpeg with desired configuration +docker_build_image: + build docker image with required dependencies pre-installed +docker_save_image: + save docker image into tar.zst +docker_load_image: + load docker image from tar.zst +docker_run_image: + run docker image with given flags +build_with_docker: + run docker image with given flags +docker_build_multiarch_image: + build multiarch docker image +efg: + estimate the film grain of a given file +encode: + encode a file using libsvtav1_psy and libopus +print_pkg_mgr: + print out evaluated package manager commands and required packages +install_deps: + install required dependencies +gen_readme: + generate project README.md +``` + + diff --git a/lib/build.sh b/lib/build.sh index 9e63071..c1cbe45 100644 --- a/lib/build.sh +++ b/lib/build.sh @@ -285,7 +285,7 @@ download_release() { done # enabling a clean build - if [[ ${CLEAN} == 'true' ]]; then + if [[ ${CLEAN} == true ]]; then DO_CLEAN="rm -rf" else DO_CLEAN='void' @@ -738,7 +738,8 @@ build_ffmpeg() { --enable-nonfree \ --disable-htmlpages \ --disable-podpages \ - --disable-txtpages || return 1 + --disable-txtpages \ + --disable-autodetect || return 1 ccache make -j"${JOBS}" || return 1 ${SUDO_MODIFY} make -j"${JOBS}" install || return 1 } diff --git a/lib/docker.sh b/lib/docker.sh index 5c3dda7..a20dfef 100644 --- a/lib/docker.sh +++ b/lib/docker.sh @@ -70,15 +70,15 @@ echo_platform() { echo "${platKernel}/${platCpu}" } -# sets DISTROS validate_selected_image() { - local selectedImage="${1:-}" + local selectedImage="$1" + local valid=1 for distro in "${VALID_DOCKER_IMAGES[@]}"; do if [[ ${selectedImage} == "${distro}" ]]; then - DISTROS+=("${distro}") + valid=0 fi done - if [[ ${DISTROS[*]} == '' ]]; then + if [[ valid -eq 1 ]]; then echo_fail "${selectedImage} is not valid" echo_info "valid images:" "${VALID_DOCKER_IMAGES[@]}" return 1 @@ -96,155 +96,168 @@ FB_FUNC_NAMES+=('docker_build_image') FB_FUNC_DESCS['docker_build_image']='build docker image with required dependencies pre-installed' FB_FUNC_COMPLETION['docker_build_image']="${VALID_DOCKER_IMAGES[*]}" docker_build_image() { - validate_selected_image "$@" || return 1 + local image="$1" + validate_selected_image "${image}" || return 1 check_docker || return 1 test -d "${DOCKER_DIR}" || mkdir -p "${DOCKER_DIR}" PLATFORM="${PLATFORM:-$(echo_platform)}" - for distro in "${DISTROS[@]}"; do - echo_info "sourcing package manager for ${distro}" - local dockerDistro="$(get_docker_image_tag "${distro}")" - # specific file for evaluated package manager info - distroPkgMgr="${DOCKER_DIR}/$(bash_basename "${distro}")-pkg_mgr" - # get package manager info - docker run \ - "${DOCKER_RUN_FLAGS[@]}" \ - "${dockerDistro}" \ - bash -c "./scripts/print_pkg_mgr.sh" | tr -d '\r' >"${distroPkgMgr}" - # shellcheck disable=SC1090 - cat "${distroPkgMgr}" - # shellcheck disable=SC1090 - source "${distroPkgMgr}" + echo_info "sourcing package manager for ${distro}" + local dockerDistro="$(get_docker_image_tag "${distro}")" + # specific file for evaluated package manager info + distroPkgMgr="${DOCKER_DIR}/$(bash_basename "${distro}")-pkg_mgr" + # get package manager info + docker run \ + "${DOCKER_RUN_FLAGS[@]}" \ + "${dockerDistro}" \ + bash -c "./scripts/print_pkg_mgr.sh" | tr -d '\r' >"${distroPkgMgr}" + # shellcheck disable=SC1090 + cat "${distroPkgMgr}" + # shellcheck disable=SC1090 + source "${distroPkgMgr}" - dockerfile="${DOCKER_DIR}/Dockerfile_$(bash_basename "${distro}")" - { - echo "FROM ${dockerDistro}" - echo 'SHELL ["/bin/bash", "-c"]' - echo 'RUN ln -sf /bin/bash /bin/sh' - echo 'ENV DEBIAN_FRONTEND=noninteractive' - # arch is rolling release, so highly likely - # an updated is required between pkg changes - if line_contains "${dockerDistro}" 'arch'; then - local archRuns='' - archRuns+="${pkg_mgr_update}" - archRuns+=" && ${pkg_mgr_upgrade}" - archRuns+=" && ${pkg_install} ${req_pkgs[*]}" - echo "RUN ${archRuns}" - else - echo "RUN ${pkg_mgr_update}" - echo "RUN ${pkg_mgr_upgrade}" - printf "RUN ${pkg_install} %s\n" "${req_pkgs[@]}" - fi - echo 'RUN pipx install virtualenv' - echo 'RUN pipx ensurepath' - echo 'ENV CARGO_HOME="/root/.cargo"' - echo 'ENV RUSTUP_HOME="/root/.rustup"' - echo 'ENV PATH="/root/.cargo/bin:$PATH"' - local cargoInst='' - cargoInst+='curl https://sh.rustup.rs -sSf | bash -s -- -y' - cargoInst+=' && rustup update stable' - cargoInst+=' && cargo install cargo-c' - cargoInst+=' && rm -rf "${CARGO_HOME}"/registry "${CARGO_HOME}"/git' - echo "RUN ${cargoInst}" - # since any user may run this image, - # open up root tools to everyone - echo 'ENV PATH="/root/.local/bin:$PATH"' - echo 'RUN chmod 777 -R /root/' - echo "WORKDIR ${DOCKER_WORKDIR}" + dockerfile="${DOCKER_DIR}/Dockerfile_$(bash_basename "${distro}")" + { + echo "FROM ${dockerDistro}" + echo 'SHELL ["/bin/bash", "-c"]' + echo 'RUN ln -sf /bin/bash /bin/sh' + echo 'ENV DEBIAN_FRONTEND=noninteractive' + # arch is rolling release, so highly likely + # an updated is required between pkg changes + if line_contains "${dockerDistro}" 'arch'; then + local archRuns='' + archRuns+="${pkg_mgr_update}" + archRuns+=" && ${pkg_mgr_upgrade}" + archRuns+=" && ${pkg_install} ${req_pkgs[*]}" + echo "RUN ${archRuns}" + else + echo "RUN ${pkg_mgr_update}" + echo "RUN ${pkg_mgr_upgrade}" + printf "RUN ${pkg_install} %s\n" "${req_pkgs[@]}" + fi + echo 'RUN pipx install virtualenv' + echo 'RUN pipx ensurepath' + echo 'ENV CARGO_HOME="/root/.cargo"' + echo 'ENV RUSTUP_HOME="/root/.rustup"' + echo 'ENV PATH="/root/.cargo/bin:$PATH"' + local cargoInst='' + cargoInst+='curl https://sh.rustup.rs -sSf | bash -s -- -y' + cargoInst+=' && rustup update stable' + cargoInst+=' && cargo install cargo-c' + cargoInst+=' && rm -rf "${CARGO_HOME}"/registry "${CARGO_HOME}"/git' + echo "RUN ${cargoInst}" + # since any user may run this image, + # open up root tools to everyone + echo 'ENV PATH="/root/.local/bin:$PATH"' + echo 'RUN chmod 777 -R /root/' + echo "WORKDIR ${DOCKER_WORKDIR}" - } >"${dockerfile}" + } >"${dockerfile}" - image_tag="$(set_distro_image_tag "${distro}")" + image_tag="$(set_distro_image_tag "${distro}")" + docker buildx build \ + --platform "${PLATFORM}" \ + -t "${image_tag}" \ + -f "${dockerfile}" \ + . || return 1 + + # if a docker registry is defined, push to it + if [[ ${DOCKER_REGISTRY} != '' ]]; then + docker_login || return 1 docker buildx build \ + --push \ --platform "${PLATFORM}" \ - -t "${image_tag}" \ + -t "${DOCKER_REGISTRY}/${image_tag}" \ -f "${dockerfile}" \ . || return 1 + fi - # if a docker registry is defined, push to it - if [[ ${DOCKER_REGISTRY} != '' ]]; then - docker_login || return 1 - docker buildx build \ - --push \ - --platform "${PLATFORM}" \ - -t "${DOCKER_REGISTRY}/${image_tag}" \ - -f "${dockerfile}" \ - . || return 1 - fi - - docker system prune -f - done + docker system prune -f } FB_FUNC_NAMES+=('docker_save_image') FB_FUNC_DESCS['docker_save_image']='save docker image into tar.zst' FB_FUNC_COMPLETION['docker_save_image']="${VALID_DOCKER_IMAGES[*]}" docker_save_image() { - validate_selected_image "$@" || return 1 + local image="$1" + validate_selected_image "${image}" || return 1 check_docker || return 1 - for distro in "${DISTROS[@]}"; do - image_tag="$(set_distro_image_tag "${distro}")" - echo_info "saving docker image for ${image_tag}" - docker save "${image_tag}" | - zstd -T0 >"${DOCKER_DIR}/$(docker_image_archive_name "${image_tag}")" || - return 1 - done + image_tag="$(set_distro_image_tag "${distro}")" + echo_info "saving docker image for ${image_tag}" + docker save "${image_tag}" | + zstd -T0 >"${DOCKER_DIR}/$(docker_image_archive_name "${image_tag}")" || + return 1 } FB_FUNC_NAMES+=('docker_load_image') FB_FUNC_DESCS['docker_load_image']='load docker image from tar.zst' FB_FUNC_COMPLETION['docker_load_image']="${VALID_DOCKER_IMAGES[*]}" docker_load_image() { - validate_selected_image "$@" || return 1 + local image="$1" + validate_selected_image "${image}" || return 1 check_docker || return 1 - for distro in "${DISTROS[@]}"; do - image_tag="$(set_distro_image_tag "${distro}")" - echo_info "loading docker image for ${image_tag}" - local archive="${DOCKER_DIR}/$(docker_image_archive_name "${image_tag}")" - test -f "$archive" || return 1 - zstdcat -T0 "$archive" | - docker load || return 1 - docker system prune -f - done + image_tag="$(set_distro_image_tag "${distro}")" + echo_info "loading docker image for ${image_tag}" + local archive="${DOCKER_DIR}/$(docker_image_archive_name "${image_tag}")" + test -f "$archive" || return 1 + zstdcat -T0 "$archive" | + docker load || return 1 + docker system prune -f } FB_FUNC_NAMES+=('docker_run_image') -FB_FUNC_DESCS['docker_run_image']='run docker image to build ffmpeg' +FB_FUNC_DESCS['docker_run_image']='run docker image with given flags' FB_FUNC_COMPLETION['docker_run_image']="${VALID_DOCKER_IMAGES[*]}" docker_run_image() { - validate_selected_image "$@" || return 1 + local image="$1" + validate_selected_image "${image}" || return 1 check_docker || return 1 - for distro in "${DISTROS[@]}"; do - dockerDistro="${distro//-/:}" - image_tag="$(set_distro_image_tag "${distro}")" + local cmd="${2}" + local runCmd=() + if [[ ${cmd} == '' ]]; then + DOCKER_RUN_FLAGS+=("-it") + else + runCmd+=(bash -c "${cmd}") + fi - # if a docker registry is defined, pull from it - if [[ ${DOCKER_REGISTRY} != '' ]]; then - docker_login || return 1 - docker pull \ - "${DOCKER_REGISTRY}/${image_tag}" || return 1 - docker tag "${DOCKER_REGISTRY}/${image_tag}" "${image_tag}" - fi + dockerDistro="${distro//-/:}" + image_tag="$(set_distro_image_tag "${distro}")" - echo_info "running ffmpeg build with ${image_tag}" - docker run \ - "${DOCKER_RUN_FLAGS[@]}" \ - -u "$(id -u):$(id -g)" \ - "${image_tag}" \ - ./scripts/build.sh || return 1 + # if a docker registry is defined, pull from it + if [[ ${DOCKER_REGISTRY} != '' ]]; then + docker_login || return 1 + docker pull \ + "${DOCKER_REGISTRY}/${image_tag}" || return 1 + docker tag "${DOCKER_REGISTRY}/${image_tag}" "${image_tag}" + fi - docker system prune -f + echo_info "running ffmpeg build with ${image_tag}" + docker run \ + "${DOCKER_RUN_FLAGS[@]}" \ + -u "$(id -u):$(id -g)" \ + "${image_tag}" \ + "${runCmd[@]}" || return 1 - return 0 - done + docker system prune -f + + return 0 +} + +FB_FUNC_NAMES+=('build_with_docker') +FB_FUNC_DESCS['build_with_docker']='run docker image with given flags' +FB_FUNC_COMPLETION['build_with_docker']="${VALID_DOCKER_IMAGES[*]}" +build_with_docker() { + local image="$1" + docker_run_image "${image}" ./scripts/build.sh } FB_FUNC_NAMES+=('docker_build_multiarch_image') FB_FUNC_DESCS['docker_build_multiarch_image']='build multiarch docker image' FB_FUNC_COMPLETION['docker_build_multiarch_image']="${VALID_DOCKER_IMAGES[*]}" docker_build_multiarch_image() { - validate_selected_image "$@" || return 1 + local image="$1" + validate_selected_image "${image}" || return 1 check_docker || return 1 PLATFORM='linux/amd64,linux/arm64' diff --git a/lib/efg.sh b/lib/efg.sh index 503c057..abb4325 100644 --- a/lib/efg.sh +++ b/lib/efg.sh @@ -175,7 +175,8 @@ efg_encode() { for ((grain = LOW; grain <= HIGH; grain += STEP)); do local file="$(bash_basename "${vid}")" local out="${EFG_DIR}/grain-${grain}-${file}" - encode -P 10 -g ${grain} -i "${vid}" "${out}" + echo_info "encoding ${file} with grain ${grain}" + echo_if_fail encode -P 10 -g ${grain} -i "${vid}" "${out}" echo -e "\tgrain: ${grain}, bitrate: $(get_avg_bitrate "${out}")" >>"${GRAIN_LOG}" rm "${out}" done @@ -200,7 +201,7 @@ efg_plot() { IFS=',' read -r grainText bitrateText <<<"${noWhite}" IFS=':' read -r _ grain <<<"${grainText}" IFS=':' read -r _ bitrate <<<"${bitrateText}" - if [[ ${setNewReference} == 'true' ]]; then + if [[ ${setNewReference} == true ]]; then referenceBitrate="${bitrate}" setNewReference=false fi @@ -248,7 +249,7 @@ efg() { GRAIN_LOG="${EFG_DIR}/${LOW}-${STEP}-${HIGH}-grains.txt" - if [[ ${PLOT} == 'true' && -f ${GRAIN_LOG} ]]; then + if [[ ${PLOT} == true && -f ${GRAIN_LOG} ]]; then efg_plot return $? fi @@ -256,7 +257,7 @@ efg() { efg_segment || return 1 efg_encode || return 1 - if [[ ${PLOT} == 'true' && -f ${GRAIN_LOG} ]]; then + if [[ ${PLOT} == true && -f ${GRAIN_LOG} ]]; then efg_plot || return 1 fi } diff --git a/lib/encode.sh b/lib/encode.sh index 74d6feb..2cfa952 100644 --- a/lib/encode.sh +++ b/lib/encode.sh @@ -381,7 +381,7 @@ gen_encode_script() { # actually do ffmpeg commmand echo - if [[ ${DISABLE_DV} == 'false' ]]; then + if [[ ${DISABLE_DV} == false ]]; then echo 'ffmpeg "${ffmpegParams[@]}" -dolbyvision 1 "${OUTPUT}" || \' fi echo 'ffmpeg "${ffmpegParams[@]}" -dolbyvision 0 "${OUTPUT}" || exit 1' @@ -398,7 +398,7 @@ gen_encode_script() { echo } >"${genScript}" - if [[ ${PRINT_OUT} == 'true' ]]; then + if [[ ${PRINT_OUT} == true ]]; then echo_info "${genScript} contents:" echo "$(<"${genScript}")" else diff --git a/lib/readme.sh b/lib/readme.sh new file mode 100644 index 0000000..5eece6d --- /dev/null +++ b/lib/readme.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +FB_FUNC_NAMES+=('gen_readme') +FB_FUNC_DESCS['gen_readme']='generate project README.md' +gen_readme() { + local readme="${REPO_DIR}/README.md" + + echo " +A collection of scripts for building ffmpeg and encoding content with the built ffmpeg + +Tested on: +- linux x86_64/aarch64 on: +$(printf ' - %s\n' "${VALID_DOCKER_IMAGES[@]}") +- darwin aarch64 + +\`\`\`bash +$(print_cmds false) +\`\`\` + +" >"${readme}" + +} diff --git a/main.sh b/main.sh index 26f6708..8e96ea5 100755 --- a/main.sh +++ b/main.sh @@ -55,14 +55,22 @@ $cmd "$@"' >"${ENTRY_SCRIPT}" FB_FUNC_NAMES+=('print_cmds') FB_FUNC_DESCS['print_cmds']='print usable commands' print_cmds() { - echo -e "\n~~~ Usable Commands ~~~" + local color="${1:-true}" + local uncolor='' + if [[ ${color} == true ]]; then + color="${CYAN}" + uncolor="${NC}" + else + color='' + fi + echo -e "~~~ Usable Commands ~~~\n" for funcname in "${FB_FUNC_NAMES[@]}"; do - echo -e "${CYAN}${funcname}${NC}:\n\t" "${FB_FUNC_DESCS[${funcname}]}" + echo -e "${color}${funcname}${uncolor}:\n\t" "${FB_FUNC_DESCS[${funcname}]}" if [[ $FB_RUNNING_AS_SCRIPT -eq 0 ]]; then (cd "$SCRIPT_DIR" && ln -sf entry.sh "${funcname}.sh") fi done - echo -e "~~~~~~~~~~~~~~~~~~~~~~~\n" + echo -e "\n" } set_completions() { diff --git a/scripts/build_with_docker.sh b/scripts/build_with_docker.sh new file mode 120000 index 0000000..44b7711 --- /dev/null +++ b/scripts/build_with_docker.sh @@ -0,0 +1 @@ +entry.sh \ No newline at end of file diff --git a/scripts/gen_readme.sh b/scripts/gen_readme.sh new file mode 120000 index 0000000..44b7711 --- /dev/null +++ b/scripts/gen_readme.sh @@ -0,0 +1 @@ +entry.sh \ No newline at end of file