Fix
This commit is contained in:
@ -0,0 +1,240 @@
|
|||||||
|
diff --git a/src/etc/poudriere.conf.sample b/src/etc/poudriere.conf.sample
|
||||||
|
index c0ba896d927ca1700cc78e57cb953deec43b5484..337eb1827904daba5bf5df5790d005f6e7fa3de9 100644
|
||||||
|
--- a/src/etc/poudriere.conf.sample
|
||||||
|
+++ b/src/etc/poudriere.conf.sample
|
||||||
|
@@ -383,3 +383,16 @@ DISTFILES_CACHE=/usr/ports/distfiles
|
||||||
|
# be fetched.
|
||||||
|
# Default: everything
|
||||||
|
#PACKAGE_FETCH_WHITELIST="gcc* rust llvm*"
|
||||||
|
+
|
||||||
|
+# Have pkg create the repo such that each package is named with the short hash
|
||||||
|
+# of its file contents in the package filename, with symlinks to the traditional
|
||||||
|
+# package filenames. The packagesite.yaml file will point to the hashed version
|
||||||
|
+# of these files. By using hashed pkg filenames, this allows users to lazily
|
||||||
|
+# cache packages without conflicting with the existing packages, or serving stale
|
||||||
|
+# packages from a cache. Once the packages are synced the much
|
||||||
|
+# smaller meta files can then be synced. Allowing a near atomic update of the repo.
|
||||||
|
+# On caching CDNs this means a need to purge 2-5 files instead of all pkgs that
|
||||||
|
+# have been updated.
|
||||||
|
+# The symlinks are only required for the local poudriere for resovling dependencies,
|
||||||
|
+# they do not need to be uploaded to the CDN.
|
||||||
|
+#PKG_HASH="no"
|
||||||
|
diff --git a/src/man/poudriere-bulk.8 b/src/man/poudriere-bulk.8
|
||||||
|
index 81cf698eeb6670048628d83e819fd84ad433be93..dac4eb3211a9fd0afe451779449693acade24430 100644
|
||||||
|
--- a/src/man/poudriere-bulk.8
|
||||||
|
+++ b/src/man/poudriere-bulk.8
|
||||||
|
@@ -28,7 +28,7 @@
|
||||||
|
.\"
|
||||||
|
.\" Note: The date here should be updated whenever a non-trivial
|
||||||
|
.\" change is made to the manual page.
|
||||||
|
-.Dd July 5, 2022
|
||||||
|
+.Dd September 26, 2022
|
||||||
|
.Dt POUDRIERE-BULK 8
|
||||||
|
.Os
|
||||||
|
.Sh NAME
|
||||||
|
@@ -38,7 +38,7 @@
|
||||||
|
.Nm
|
||||||
|
.Fl a
|
||||||
|
.Fl j Ar name
|
||||||
|
-.Op Fl CcFIikNnRrSTtvw
|
||||||
|
+.Op Fl CcFHIikNnRrSTtvw
|
||||||
|
.Op Fl B Ar name
|
||||||
|
.Op Fl b Ar branch
|
||||||
|
.Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
|
||||||
|
@@ -48,7 +48,7 @@
|
||||||
|
.Nm
|
||||||
|
.Fl f Ar file Op Fl f Ar file2 Ar ...
|
||||||
|
.Fl j Ar name
|
||||||
|
-.Op Fl CcFIikNnRrSTtvw
|
||||||
|
+.Op Fl CcFHIikNnRrSTtvw
|
||||||
|
.Op Fl B Ar name
|
||||||
|
.Op Fl b Ar branch
|
||||||
|
.Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
|
||||||
|
@@ -57,7 +57,7 @@
|
||||||
|
.Op Fl z Ar set
|
||||||
|
.Nm
|
||||||
|
.Fl j Ar name
|
||||||
|
-.Op Fl CcFIikNnRrSTtvw
|
||||||
|
+.Op Fl CcFHIikNnRrSTtvw
|
||||||
|
.Op Fl B Ar name
|
||||||
|
.Op Fl b Ar branch
|
||||||
|
.Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
|
||||||
|
@@ -232,6 +232,8 @@ Fetch only from the original
|
||||||
|
Skip
|
||||||
|
.Fx
|
||||||
|
mirrors.
|
||||||
|
+.It Fl H
|
||||||
|
+Create a repository where the package filenames contain the short hash of the contents.
|
||||||
|
.It Fl I
|
||||||
|
Advanced interactive mode.
|
||||||
|
.Pp
|
||||||
|
diff --git a/src/share/poudriere/bulk.sh b/src/share/poudriere/bulk.sh
|
||||||
|
index 3507ceb3548910c8aa688148614f1eadf72dd695..126b3becc6d050ede819bddebc0ae55b6712aaba 100755
|
||||||
|
--- a/src/share/poudriere/bulk.sh
|
||||||
|
+++ b/src/share/poudriere/bulk.sh
|
||||||
|
@@ -47,6 +47,8 @@ Options:
|
||||||
|
-f file. Implies -c for -a.
|
||||||
|
-c -- Clean all the previously built binary packages and logs.
|
||||||
|
-F -- Only fetch from original master_site (skip FreeBSD mirrors)
|
||||||
|
+ -H -- Create a repository where the package filenames contain the
|
||||||
|
+ short hash of the contents.
|
||||||
|
-I -- Advanced Interactive mode. Leaves jail running with ports
|
||||||
|
installed after test.
|
||||||
|
-i -- Interactive mode. Enter jail for interactive testing and
|
||||||
|
@@ -97,7 +99,7 @@ if [ $# -eq 0 ]; then
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
|
||||||
|
-while getopts "ab:B:CcFf:iIj:J:knNO:p:RrSTtvwz:" FLAG; do
|
||||||
|
+while getopts "ab:B:CcFf:HiIj:J:knNO:p:RrSTtvwz:" FLAG; do
|
||||||
|
case "${FLAG}" in
|
||||||
|
a)
|
||||||
|
ALL=1
|
||||||
|
@@ -126,6 +128,9 @@ while getopts "ab:B:CcFf:iIj:J:knNO:p:RrSTtvwz:" FLAG; do
|
||||||
|
fi
|
||||||
|
LISTPKGS="${LISTPKGS:+${LISTPKGS} }${OPTARG}"
|
||||||
|
;;
|
||||||
|
+ H)
|
||||||
|
+ PKG_REPO_FLAGS="${PKG_REPO_FLAGS:+${PKG_REPO_FLAGS} }--hash --symlink"
|
||||||
|
+ ;;
|
||||||
|
I)
|
||||||
|
INTERACTIVE_MODE=2
|
||||||
|
;;
|
||||||
|
diff --git a/src/share/poudriere/common.sh b/src/share/poudriere/common.sh
|
||||||
|
index a4907362c76000bc81fb8b7dbd61918fb7426538..e52cfb432e15783e9d3b47a018d8d1735a70a5cf 100755
|
||||||
|
--- a/src/share/poudriere/common.sh
|
||||||
|
+++ b/src/share/poudriere/common.sh
|
||||||
|
@@ -6473,6 +6473,10 @@ delete_old_pkg() {
|
||||||
|
if [ -L "${pkg}" ]; then
|
||||||
|
is_sym=1
|
||||||
|
fi
|
||||||
|
+ if [ -d "${pkg}" ] && [ "${pkgfile}" = "Hashed" ]; then
|
||||||
|
+ msg_debug "Ignoring directory"
|
||||||
|
+ return 0;
|
||||||
|
+ fi
|
||||||
|
if [ "${is_sym}" -eq 1 ] && [ ! -e "${pkg}" ]; then
|
||||||
|
msg "Deleting ${COLOR_PORT}${pkgfile}${COLOR_RESET}: dead symlink"
|
||||||
|
delete_pkg "${pkg}"
|
||||||
|
@@ -9468,12 +9472,16 @@ clean_restricted() {
|
||||||
|
}
|
||||||
|
|
||||||
|
build_repo() {
|
||||||
|
- local origin pkg_repo_list_files
|
||||||
|
+ local origin pkg_repo_list_files hashcmd
|
||||||
|
|
||||||
|
msg "Creating pkg repository"
|
||||||
|
if [ ${DRY_RUN} -eq 1 ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
+ if [ ${PKG_HASH} != "no" ]; then
|
||||||
|
+ hashcmd="--hash --symlink"
|
||||||
|
+ PKG_REPO_FLAGS="${PKG_REPO_FLAGS:+${PKG_REPO_FLAGS} }$hashcmd"
|
||||||
|
+ fi
|
||||||
|
bset status "pkgrepo:"
|
||||||
|
ensure_pkg_installed force_extract || \
|
||||||
|
err 1 "Unable to extract pkg."
|
||||||
|
@@ -9493,12 +9501,20 @@ build_repo() {
|
||||||
|
install -m 0400 "${PKG_REPO_META_FILE}" \
|
||||||
|
"${MASTERMNT:?}/tmp/pkgmeta"
|
||||||
|
fi
|
||||||
|
+
|
||||||
|
+ # Remount rw
|
||||||
|
+ # mount_nullfs does not support mount -u
|
||||||
|
+ umount ${UMOUNT_NONBUSY} ${MASTERMNT}/packages || \
|
||||||
|
+ umount -f ${MASTERMNT}/packages
|
||||||
|
+ mount_packages
|
||||||
|
+
|
||||||
|
mkdir -p ${MASTERMNT}/tmp/packages
|
||||||
|
if [ -n "${PKG_REPO_SIGNING_KEY}" ]; then
|
||||||
|
msg "Signing repository with key: ${PKG_REPO_SIGNING_KEY}"
|
||||||
|
install -m 0400 "${PKG_REPO_SIGNING_KEY}" \
|
||||||
|
"${MASTERMNT:?}/tmp/repo.key"
|
||||||
|
injail ${PKG_BIN:?} repo \
|
||||||
|
+ ${PKG_REPO_FLAGS} \
|
||||||
|
${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
|
||||||
|
-o /tmp/packages \
|
||||||
|
${PKG_META} \
|
||||||
|
@@ -9515,6 +9531,7 @@ build_repo() {
|
||||||
|
# using SSH with DNSSEC as older hosts don't support
|
||||||
|
# it.
|
||||||
|
${MASTERMNT:?}${PKG_BIN:?} repo \
|
||||||
|
+ ${PKG_REPO_FLAGS} \
|
||||||
|
${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
|
||||||
|
-o "${MASTERMNT:?}/tmp/packages" ${PKG_META_MASTERMNT} \
|
||||||
|
"${MASTERMNT:?}/packages" \
|
||||||
|
@@ -9527,6 +9544,7 @@ build_repo() {
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
JNETNAME="n" injail ${PKG_BIN:?} repo \
|
||||||
|
+ ${PKG_REPO_FLAGS} \
|
||||||
|
${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
|
||||||
|
-o /tmp/packages ${PKG_META} /packages \
|
||||||
|
${SIGNING_COMMAND:+signing_command: ${SIGNING_COMMAND}} ||
|
||||||
|
@@ -9542,6 +9560,11 @@ build_repo() {
|
||||||
|
sign_pkg pubkey "${PACKAGES:?}/Latest/pkg.${PKG_EXT}"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
+
|
||||||
|
+ # Remount ro
|
||||||
|
+ umount ${UMOUNT_NONBUSY} ${MASTERMNT}/packages || \
|
||||||
|
+ umount -f ${MASTERMNT}/packages
|
||||||
|
+ mount_packages -o ro
|
||||||
|
}
|
||||||
|
|
||||||
|
calculate_size_in_mb() {
|
||||||
|
@@ -10109,6 +10132,7 @@ esac
|
||||||
|
: ${FLAVOR_DEFAULT_ALL:=no}
|
||||||
|
: ${NULLFS_PATHS:="/rescue /usr/share /usr/tests /usr/lib32"}
|
||||||
|
: ${PACKAGE_FETCH_URL:="pkg+http://pkg.FreeBSD.org/\${ABI}"}
|
||||||
|
+: ${PKG_HASH:=no}
|
||||||
|
|
||||||
|
: ${POUDRIERE_TMPDIR:=$(command mktemp -dt poudriere)}
|
||||||
|
: ${SHASH_VAR_PATH_DEFAULT:=${POUDRIERE_TMPDIR}}
|
||||||
|
diff --git a/src/share/poudriere/include/pkg.sh b/src/share/poudriere/include/pkg.sh
|
||||||
|
index d4821400cb2ad20a6994a5d9145604f8081a98f6..360518520d6f1067531380ea9040ec2d1d4c1f33 100644
|
||||||
|
--- a/src/share/poudriere/include/pkg.sh
|
||||||
|
+++ b/src/share/poudriere/include/pkg.sh
|
||||||
|
@@ -400,6 +400,10 @@ delete_pkg() {
|
||||||
|
local pkg="$1"
|
||||||
|
|
||||||
|
clear_pkg_cache "${pkg}"
|
||||||
|
+
|
||||||
|
+ # If ${pkg} is a symlink, delete the target as well
|
||||||
|
+ [ -L "${pkg}" ] && unlink $(realpath "${pkg}")
|
||||||
|
+
|
||||||
|
# Delete the package and the depsfile since this package is being deleted,
|
||||||
|
# which will force it to be recreated
|
||||||
|
unlink "${pkg}"
|
||||||
|
@@ -417,6 +421,8 @@ delete_pkg_xargs() {
|
||||||
|
# Delete the package and the depsfile since this package is being deleted,
|
||||||
|
# which will force it to be recreated
|
||||||
|
{
|
||||||
|
+ # If ${pkg} is a symlink, delete the target as well
|
||||||
|
+ [ -L "${pkg}" ] && echo $(realpath "${pkg}")
|
||||||
|
echo "${pkg}"
|
||||||
|
echo "${pkg_cache_dir}"
|
||||||
|
} >> "${listfile}"
|
||||||
|
diff --git a/src/share/poudriere/pkgclean.sh b/src/share/poudriere/pkgclean.sh
|
||||||
|
old mode 100755
|
||||||
|
new mode 100644
|
||||||
|
index fafc3c0a547335ddee32033bf9d0700ccf556fca..5458d3d04dd2e05727c1c79ce0ba0d73d3946665
|
||||||
|
--- a/src/share/poudriere/pkgclean.sh
|
||||||
|
+++ b/src/share/poudriere/pkgclean.sh
|
||||||
|
@@ -289,6 +289,8 @@ check_should_delete_pkg() {
|
||||||
|
*".${PKG_EXT}")
|
||||||
|
if should_delete "${file}"; then
|
||||||
|
echo "${file}" >> "${BADFILES_LIST:?}"
|
||||||
|
+ # If the pkg is a symlink to a hashed package, remove the hashed version as well
|
||||||
|
+ [ -L "${file}" ] && echo "$(realpath ${file})" >> ${BADFILES_LIST}
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
*.txz)
|
||||||
|
@@ -305,6 +307,8 @@ check_should_delete_pkg() {
|
||||||
|
*)
|
||||||
|
msg_verbose "Found incorrect format file: ${file}"
|
||||||
|
echo "${file}" >> "${BADFILES_LIST:?}"
|
||||||
|
+ # If the pkg is a symlink to a hashed package, remove the hashed version as well
|
||||||
|
+ [ -L "${file}" ] && echo "$(realpath ${file})" >> ${BADFILES_LIST}
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user