diff --git a/ports-mgmt/poudriere-devel/6cd972994ddd6cca1376260dbd2675eb77187ae8.diff b/ports-mgmt/poudriere-devel/6cd972994ddd6cca1376260dbd2675eb77187ae8.diff new file mode 100644 index 0000000..652a896 --- /dev/null +++ b/ports-mgmt/poudriere-devel/6cd972994ddd6cca1376260dbd2675eb77187ae8.diff @@ -0,0 +1,240 @@ +diff --git a/src/etc/poudriere.conf.sample b/src/etc/poudriere.conf.sample +index c0ba896d927ca1700cc78e57cb953deec43b5484..337eb1827904daba5bf5df5790d005f6e7fa3de9 100644 +--- a/src/etc/poudriere.conf.sample ++++ b/src/etc/poudriere.conf.sample +@@ -383,3 +383,16 @@ DISTFILES_CACHE=/usr/ports/distfiles + # be fetched. + # Default: everything + #PACKAGE_FETCH_WHITELIST="gcc* rust llvm*" ++ ++# Have pkg create the repo such that each package is named with the short hash ++# of its file contents in the package filename, with symlinks to the traditional ++# package filenames. The packagesite.yaml file will point to the hashed version ++# of these files. By using hashed pkg filenames, this allows users to lazily ++# cache packages without conflicting with the existing packages, or serving stale ++# packages from a cache. Once the packages are synced the much ++# smaller meta files can then be synced. Allowing a near atomic update of the repo. ++# On caching CDNs this means a need to purge 2-5 files instead of all pkgs that ++# have been updated. ++# The symlinks are only required for the local poudriere for resovling dependencies, ++# they do not need to be uploaded to the CDN. ++#PKG_HASH="no" +diff --git a/src/man/poudriere-bulk.8 b/src/man/poudriere-bulk.8 +index 81cf698eeb6670048628d83e819fd84ad433be93..dac4eb3211a9fd0afe451779449693acade24430 100644 +--- a/src/man/poudriere-bulk.8 ++++ b/src/man/poudriere-bulk.8 +@@ -28,7 +28,7 @@ + .\" + .\" Note: The date here should be updated whenever a non-trivial + .\" change is made to the manual page. +-.Dd July 5, 2022 ++.Dd September 26, 2022 + .Dt POUDRIERE-BULK 8 + .Os + .Sh NAME +@@ -38,7 +38,7 @@ + .Nm + .Fl a + .Fl j Ar name +-.Op Fl CcFIikNnRrSTtvw ++.Op Fl CcFHIikNnRrSTtvw + .Op Fl B Ar name + .Op Fl b Ar branch + .Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs +@@ -48,7 +48,7 @@ + .Nm + .Fl f Ar file Op Fl f Ar file2 Ar ... + .Fl j Ar name +-.Op Fl CcFIikNnRrSTtvw ++.Op Fl CcFHIikNnRrSTtvw + .Op Fl B Ar name + .Op Fl b Ar branch + .Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs +@@ -57,7 +57,7 @@ + .Op Fl z Ar set + .Nm + .Fl j Ar name +-.Op Fl CcFIikNnRrSTtvw ++.Op Fl CcFHIikNnRrSTtvw + .Op Fl B Ar name + .Op Fl b Ar branch + .Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs +@@ -232,6 +232,8 @@ Fetch only from the original + Skip + .Fx + mirrors. ++.It Fl H ++Create a repository where the package filenames contain the short hash of the contents. + .It Fl I + Advanced interactive mode. + .Pp +diff --git a/src/share/poudriere/bulk.sh b/src/share/poudriere/bulk.sh +index 3507ceb3548910c8aa688148614f1eadf72dd695..126b3becc6d050ede819bddebc0ae55b6712aaba 100755 +--- a/src/share/poudriere/bulk.sh ++++ b/src/share/poudriere/bulk.sh +@@ -47,6 +47,8 @@ Options: + -f file. Implies -c for -a. + -c -- Clean all the previously built binary packages and logs. + -F -- Only fetch from original master_site (skip FreeBSD mirrors) ++ -H -- Create a repository where the package filenames contain the ++ short hash of the contents. + -I -- Advanced Interactive mode. Leaves jail running with ports + installed after test. + -i -- Interactive mode. Enter jail for interactive testing and +@@ -97,7 +99,7 @@ if [ $# -eq 0 ]; then + usage + fi + +-while getopts "ab:B:CcFf:iIj:J:knNO:p:RrSTtvwz:" FLAG; do ++while getopts "ab:B:CcFf:HiIj:J:knNO:p:RrSTtvwz:" FLAG; do + case "${FLAG}" in + a) + ALL=1 +@@ -126,6 +128,9 @@ while getopts "ab:B:CcFf:iIj:J:knNO:p:RrSTtvwz:" FLAG; do + fi + LISTPKGS="${LISTPKGS:+${LISTPKGS} }${OPTARG}" + ;; ++ H) ++ PKG_REPO_FLAGS="${PKG_REPO_FLAGS:+${PKG_REPO_FLAGS} }--hash --symlink" ++ ;; + I) + INTERACTIVE_MODE=2 + ;; +diff --git a/src/share/poudriere/common.sh b/src/share/poudriere/common.sh +index a4907362c76000bc81fb8b7dbd61918fb7426538..e52cfb432e15783e9d3b47a018d8d1735a70a5cf 100755 +--- a/src/share/poudriere/common.sh ++++ b/src/share/poudriere/common.sh +@@ -6473,6 +6473,10 @@ delete_old_pkg() { + if [ -L "${pkg}" ]; then + is_sym=1 + fi ++ if [ -d "${pkg}" ] && [ "${pkgfile}" = "Hashed" ]; then ++ msg_debug "Ignoring directory" ++ return 0; ++ fi + if [ "${is_sym}" -eq 1 ] && [ ! -e "${pkg}" ]; then + msg "Deleting ${COLOR_PORT}${pkgfile}${COLOR_RESET}: dead symlink" + delete_pkg "${pkg}" +@@ -9468,12 +9472,16 @@ clean_restricted() { + } + + build_repo() { +- local origin pkg_repo_list_files ++ local origin pkg_repo_list_files hashcmd + + msg "Creating pkg repository" + if [ ${DRY_RUN} -eq 1 ]; then + return 0 + fi ++ if [ ${PKG_HASH} != "no" ]; then ++ hashcmd="--hash --symlink" ++ PKG_REPO_FLAGS="${PKG_REPO_FLAGS:+${PKG_REPO_FLAGS} }$hashcmd" ++ fi + bset status "pkgrepo:" + ensure_pkg_installed force_extract || \ + err 1 "Unable to extract pkg." +@@ -9493,12 +9501,20 @@ build_repo() { + install -m 0400 "${PKG_REPO_META_FILE}" \ + "${MASTERMNT:?}/tmp/pkgmeta" + fi ++ ++ # Remount rw ++ # mount_nullfs does not support mount -u ++ umount ${UMOUNT_NONBUSY} ${MASTERMNT}/packages || \ ++ umount -f ${MASTERMNT}/packages ++ mount_packages ++ + mkdir -p ${MASTERMNT}/tmp/packages + if [ -n "${PKG_REPO_SIGNING_KEY}" ]; then + msg "Signing repository with key: ${PKG_REPO_SIGNING_KEY}" + install -m 0400 "${PKG_REPO_SIGNING_KEY}" \ + "${MASTERMNT:?}/tmp/repo.key" + injail ${PKG_BIN:?} repo \ ++ ${PKG_REPO_FLAGS} \ + ${pkg_repo_list_files:+"${pkg_repo_list_files}"} \ + -o /tmp/packages \ + ${PKG_META} \ +@@ -9515,6 +9531,7 @@ build_repo() { + # using SSH with DNSSEC as older hosts don't support + # it. + ${MASTERMNT:?}${PKG_BIN:?} repo \ ++ ${PKG_REPO_FLAGS} \ + ${pkg_repo_list_files:+"${pkg_repo_list_files}"} \ + -o "${MASTERMNT:?}/tmp/packages" ${PKG_META_MASTERMNT} \ + "${MASTERMNT:?}/packages" \ +@@ -9527,6 +9544,7 @@ build_repo() { + ;; + esac + JNETNAME="n" injail ${PKG_BIN:?} repo \ ++ ${PKG_REPO_FLAGS} \ + ${pkg_repo_list_files:+"${pkg_repo_list_files}"} \ + -o /tmp/packages ${PKG_META} /packages \ + ${SIGNING_COMMAND:+signing_command: ${SIGNING_COMMAND}} || +@@ -9542,6 +9560,11 @@ build_repo() { + sign_pkg pubkey "${PACKAGES:?}/Latest/pkg.${PKG_EXT}" + fi + fi ++ ++ # Remount ro ++ umount ${UMOUNT_NONBUSY} ${MASTERMNT}/packages || \ ++ umount -f ${MASTERMNT}/packages ++ mount_packages -o ro + } + + calculate_size_in_mb() { +@@ -10109,6 +10132,7 @@ esac + : ${FLAVOR_DEFAULT_ALL:=no} + : ${NULLFS_PATHS:="/rescue /usr/share /usr/tests /usr/lib32"} + : ${PACKAGE_FETCH_URL:="pkg+http://pkg.FreeBSD.org/\${ABI}"} ++: ${PKG_HASH:=no} + + : ${POUDRIERE_TMPDIR:=$(command mktemp -dt poudriere)} + : ${SHASH_VAR_PATH_DEFAULT:=${POUDRIERE_TMPDIR}} +diff --git a/src/share/poudriere/include/pkg.sh b/src/share/poudriere/include/pkg.sh +index d4821400cb2ad20a6994a5d9145604f8081a98f6..360518520d6f1067531380ea9040ec2d1d4c1f33 100644 +--- a/src/share/poudriere/include/pkg.sh ++++ b/src/share/poudriere/include/pkg.sh +@@ -400,6 +400,10 @@ delete_pkg() { + local pkg="$1" + + clear_pkg_cache "${pkg}" ++ ++ # If ${pkg} is a symlink, delete the target as well ++ [ -L "${pkg}" ] && unlink $(realpath "${pkg}") ++ + # Delete the package and the depsfile since this package is being deleted, + # which will force it to be recreated + unlink "${pkg}" +@@ -417,6 +421,8 @@ delete_pkg_xargs() { + # Delete the package and the depsfile since this package is being deleted, + # which will force it to be recreated + { ++ # If ${pkg} is a symlink, delete the target as well ++ [ -L "${pkg}" ] && echo $(realpath "${pkg}") + echo "${pkg}" + echo "${pkg_cache_dir}" + } >> "${listfile}" +diff --git a/src/share/poudriere/pkgclean.sh b/src/share/poudriere/pkgclean.sh +old mode 100755 +new mode 100644 +index fafc3c0a547335ddee32033bf9d0700ccf556fca..5458d3d04dd2e05727c1c79ce0ba0d73d3946665 +--- a/src/share/poudriere/pkgclean.sh ++++ b/src/share/poudriere/pkgclean.sh +@@ -289,6 +289,8 @@ check_should_delete_pkg() { + *".${PKG_EXT}") + if should_delete "${file}"; then + echo "${file}" >> "${BADFILES_LIST:?}" ++ # If the pkg is a symlink to a hashed package, remove the hashed version as well ++ [ -L "${file}" ] && echo "$(realpath ${file})" >> ${BADFILES_LIST} + fi + ;; + *.txz) +@@ -305,6 +307,8 @@ check_should_delete_pkg() { + *) + msg_verbose "Found incorrect format file: ${file}" + echo "${file}" >> "${BADFILES_LIST:?}" ++ # If the pkg is a symlink to a hashed package, remove the hashed version as well ++ [ -L "${file}" ] && echo "$(realpath ${file})" >> ${BADFILES_LIST} + ;; + esac + }