kiwi/klara-ports
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix ptaches

Browse Source
This commit is contained in:
Xavier Beaudouin
2024-08-26 12:02:49 +02:00
parent 276c36772e
commit 611a04f2fb
7 changed files with 217 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
ports-mgmt/poudriere-devel/files/patch-foo.diff
View File

@ -1,232 +0,0 @@
diff --git src/etc/poudriere.conf.sample src/etc/poudriere.conf.sample
--- src/etc/poudriere.conf.sample
+++ src/etc/poudriere.conf.sample
@@ -383,3 +383,16 @@ DISTFILES_CACHE=/usr/ports/distfiles
# be fetched.
# Default: everything
#PACKAGE_FETCH_WHITELIST="gcc* rust llvm*"
+
+# Have pkg create the repo such that each package is named with the short hash
+# of its file contents in the package filename, with symlinks to the traditional
+# package filenames. The packagesite.yaml file will point to the hashed version
+# of these files. By using hashed pkg filenames, this allows users to lazily
+# cache packages without conflicting with the existing packages, or serving stale
+# packages from a cache. Once the packages are synced the much
+# smaller meta files can then be synced. Allowing a near atomic update of the repo.
+# On caching CDNs this means a need to purge 2-5 files instead of all pkgs that
+# have been updated.
+# The symlinks are only required for the local poudriere for resovling dependencies,
+# they do not need to be uploaded to the CDN.
+#PKG_HASH="no"
diff --git src/man/poudriere-bulk.8 src/man/poudriere-bulk.8
--- src/man/poudriere-bulk.8
+++ src/man/poudriere-bulk.8
@@ -28,7 +28,7 @@
.\"
.\" Note: The date here should be updated whenever a non-trivial
.\" change is made to the manual page.
-.Dd July 5, 2022
+.Dd September 26, 2022
.Dt POUDRIERE-BULK 8
.Os
.Sh NAME
@@ -38,7 +38,7 @@
.Nm
.Fl a
.Fl j Ar name
-.Op Fl CcFIikNnRrSTtvw
+.Op Fl CcFHIikNnRrSTtvw
.Op Fl B Ar name
.Op Fl b Ar branch
.Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
@@ -48,7 +48,7 @@
.Nm
.Fl f Ar file Op Fl f Ar file2 Ar ...
.Fl j Ar name
-.Op Fl CcFIikNnRrSTtvw
+.Op Fl CcFHIikNnRrSTtvw
.Op Fl B Ar name
.Op Fl b Ar branch
.Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
@@ -57,7 +57,7 @@
.Op Fl z Ar set
.Nm
.Fl j Ar name
-.Op Fl CcFIikNnRrSTtvw
+.Op Fl CcFHIikNnRrSTtvw
.Op Fl B Ar name
.Op Fl b Ar branch
.Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
@@ -232,6 +232,8 @@ Fetch only from the original
Skip
.Fx
mirrors.
+.It Fl H
+Create a repository where the package filenames contain the short hash of the contents.
.It Fl I
Advanced interactive mode.
.Pp
diff --git src/share/poudriere/bulk.sh src/share/poudriere/bulk.sh
--- src/share/poudriere/bulk.sh
+++ src/share/poudriere/bulk.sh
@@ -47,6 +47,8 @@ Options:
-f file. Implies -c for -a.
-c -- Clean all the previously built binary packages and logs.
-F -- Only fetch from original master_site (skip FreeBSD mirrors)
+ -H -- Create a repository where the package filenames contain the
+ short hash of the contents.
-I -- Advanced Interactive mode. Leaves jail running with ports
installed after test.
-i -- Interactive mode. Enter jail for interactive testing and
@@ -97,7 +99,7 @@ if [ $# -eq 0 ]; then
usage
fi
-while getopts "ab:B:CcFf:iIj:J:knNO:p:RrSTtvwz:" FLAG; do
+while getopts "ab:B:CcFf:HiIj:J:knNO:p:RrSTtvwz:" FLAG; do
case "${FLAG}" in
a)
ALL=1
@@ -126,6 +128,9 @@ while getopts "ab:B:CcFf:iIj:J:knNO:p:RrSTtvwz:" FLAG; do
fi
LISTPKGS="${LISTPKGS:+${LISTPKGS} }${OPTARG}"
;;
+ H)
+ PKG_REPO_FLAGS="${PKG_REPO_FLAGS:+${PKG_REPO_FLAGS} }--hash --symlink"
+ ;;
I)
INTERACTIVE_MODE=2
;;
diff --git src/share/poudriere/common.sh src/share/poudriere/common.sh
--- src/share/poudriere/common.sh
+++ src/share/poudriere/common.sh
@@ -6473,6 +6473,10 @@ delete_old_pkg() {
if [ -L "${pkg}" ]; then
is_sym=1
fi
+ if [ -d "${pkg}" ] && [ "${pkgfile}" = "Hashed" ]; then
+ msg_debug "Ignoring directory"
+ return 0;
+ fi
if [ "${is_sym}" -eq 1 ] && [ ! -e "${pkg}" ]; then
msg "Deleting ${COLOR_PORT}${pkgfile}${COLOR_RESET}: dead symlink"
delete_pkg "${pkg}"
@@ -9468,12 +9472,16 @@ clean_restricted() {
}
build_repo() {
- local origin pkg_repo_list_files
+ local origin pkg_repo_list_files hashcmd
msg "Creating pkg repository"
if [ ${DRY_RUN} -eq 1 ]; then
return 0
fi
+ if [ ${PKG_HASH} != "no" ]; then
+ hashcmd="--hash --symlink"
+ PKG_REPO_FLAGS="${PKG_REPO_FLAGS:+${PKG_REPO_FLAGS} }$hashcmd"
+ fi
bset status "pkgrepo:"
ensure_pkg_installed force_extract || \
err 1 "Unable to extract pkg."
@@ -9493,12 +9501,20 @@ build_repo() {
install -m 0400 "${PKG_REPO_META_FILE}" \
"${MASTERMNT:?}/tmp/pkgmeta"
fi
+
+ # Remount rw
+ # mount_nullfs does not support mount -u
+ umount ${UMOUNT_NONBUSY} ${MASTERMNT}/packages || \
+ umount -f ${MASTERMNT}/packages
+ mount_packages
+
mkdir -p ${MASTERMNT}/tmp/packages
if [ -n "${PKG_REPO_SIGNING_KEY}" ]; then
msg "Signing repository with key: ${PKG_REPO_SIGNING_KEY}"
install -m 0400 "${PKG_REPO_SIGNING_KEY}" \
"${MASTERMNT:?}/tmp/repo.key"
injail ${PKG_BIN:?} repo \
+ ${PKG_REPO_FLAGS} \
${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
-o /tmp/packages \
${PKG_META} \
@@ -9515,6 +9531,7 @@ build_repo() {
# using SSH with DNSSEC as older hosts don't support
# it.
${MASTERMNT:?}${PKG_BIN:?} repo \
+ ${PKG_REPO_FLAGS} \
${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
-o "${MASTERMNT:?}/tmp/packages" ${PKG_META_MASTERMNT} \
"${MASTERMNT:?}/packages" \
@@ -9527,6 +9544,7 @@ build_repo() {
;;
esac
JNETNAME="n" injail ${PKG_BIN:?} repo \
+ ${PKG_REPO_FLAGS} \
${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
-o /tmp/packages ${PKG_META} /packages \
${SIGNING_COMMAND:+signing_command: ${SIGNING_COMMAND}} ||
@@ -9542,6 +9560,11 @@ build_repo() {
sign_pkg pubkey "${PACKAGES:?}/Latest/pkg.${PKG_EXT}"
fi
fi
+
+ # Remount ro
+ umount ${UMOUNT_NONBUSY} ${MASTERMNT}/packages || \
+ umount -f ${MASTERMNT}/packages
+ mount_packages -o ro
}
calculate_size_in_mb() {
@@ -10109,6 +10132,7 @@ esac
: ${FLAVOR_DEFAULT_ALL:=no}
: ${NULLFS_PATHS:="/rescue /usr/share /usr/tests /usr/lib32"}
: ${PACKAGE_FETCH_URL:="pkg+http://pkg.FreeBSD.org/\${ABI}"}
+: ${PKG_HASH:=no}
: ${POUDRIERE_TMPDIR:=$(command mktemp -dt poudriere)}
: ${SHASH_VAR_PATH_DEFAULT:=${POUDRIERE_TMPDIR}}
diff --git src/share/poudriere/include/pkg.sh src/share/poudriere/include/pkg.sh
--- src/share/poudriere/include/pkg.sh
+++ src/share/poudriere/include/pkg.sh
@@ -400,6 +400,10 @@ delete_pkg() {
local pkg="$1"
clear_pkg_cache "${pkg}"
+
+ # If ${pkg} is a symlink, delete the target as well
+ [ -L "${pkg}" ] && unlink $(realpath "${pkg}")
+
# Delete the package and the depsfile since this package is being deleted,
# which will force it to be recreated
unlink "${pkg}"
@@ -417,6 +421,8 @@ delete_pkg_xargs() {
# Delete the package and the depsfile since this package is being deleted,
# which will force it to be recreated
{
+ # If ${pkg} is a symlink, delete the target as well
+ [ -L "${pkg}" ] && echo $(realpath "${pkg}")
echo "${pkg}"
echo "${pkg_cache_dir}"
} >> "${listfile}"
diff --git src/share/poudriere/pkgclean.sh src/share/poudriere/pkgclean.sh
--- src/share/poudriere/pkgclean.sh
+++ src/share/poudriere/pkgclean.sh
@@ -289,6 +289,8 @@ check_should_delete_pkg() {
*".${PKG_EXT}")
if should_delete "${file}"; then
echo "${file}" >> "${BADFILES_LIST:?}"
+ # If the pkg is a symlink to a hashed package, remove the hashed version as well
+ [ -L "${file}" ] && echo "$(realpath ${file})" >> ${BADFILES_LIST}
fi
;;
*.txz)
@@ -305,6 +307,8 @@ check_should_delete_pkg() {
*)
msg_verbose "Found incorrect format file: ${file}"
echo "${file}" >> "${BADFILES_LIST:?}"
+ # If the pkg is a symlink to a hashed package, remove the hashed version as well
+ [ -L "${file}" ] && echo "$(realpath ${file})" >> ${BADFILES_LIST}
;;
esac
}

19
ports-mgmt/poudriere-devel/files/patch-src_etc_poudriere.conf.sample Normal file
View File

@ -0,0 +1,19 @@
--- src/etc/poudriere.conf.sample.orig 2024-08-11 06:43:13 UTC
+++ src/etc/poudriere.conf.sample
@@ -385,3 +385,16 @@ DISTFILES_CACHE=/usr/ports/distfiles
# be fetched.
# Default: everything
#PACKAGE_FETCH_WHITELIST="gcc* rust llvm*"
+
+# Have pkg create the repo such that each package is named with the short hash
+# of its file contents in the package filename, with symlinks to the traditional
+# package filenames. The packagesite.yaml file will point to the hashed version
+# of these files. By using hashed pkg filenames, this allows users to lazily
+# cache packages without conflicting with the existing packages, or serving stale
+# packages from a cache. Once the packages are synced the much
+# smaller meta files can then be synced. Allowing a near atomic update of the repo.
+# On caching CDNs this means a need to purge 2-5 files instead of all pkgs that
+# have been updated.
+# The symlinks are only required for the local poudriere for resovling dependencies,
+# they do not need to be uploaded to the CDN.
+#PKG_HASH="no"

38
ports-mgmt/poudriere-devel/files/patch-src_man_poudriere-bulk.8 Normal file
View File

@ -0,0 +1,38 @@
--- src/man/poudriere-bulk.8.orig 2024-08-11 06:43:13 UTC
+++ src/man/poudriere-bulk.8
@@ -38,7 +38,7 @@
.Nm
.Fl a
.Fl j Ar name
-.Op Fl CcFIikNnRrSTtvw
+.Op Fl CcFHIikNnRrSTtvw
.Op Fl B Ar name
.Op Fl b Ar branch
.Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
@@ -48,7 +48,7 @@
.Nm
.Fl f Ar file Op Fl f Ar file2 Ar ...
.Fl j Ar name
-.Op Fl CcFIikNnRrSTtvw
+.Op Fl CcFHIikNnRrSTtvw
.Op Fl B Ar name
.Op Fl b Ar branch
.Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
@@ -57,7 +57,7 @@
.Op Fl z Ar set
.Nm
.Fl j Ar name
-.Op Fl CcFIikNnRrSTtvw
+.Op Fl CcFHIikNnRrSTtvw
.Op Fl B Ar name
.Op Fl b Ar branch
.Op Fl J Ar maxjobs Ns Op Cm \&: Ns Ar prebuildmaxjobs
@@ -232,6 +232,8 @@ mirrors.
Skip
.Fx
mirrors.
+.It Fl H
+Create a repository where the package filenames contain the short hash of the contents.
.It Fl I
Advanced interactive mode.
.Pp

30
ports-mgmt/poudriere-devel/files/patch-src_share_poudriere_bulk.sh Normal file
View File

@ -0,0 +1,30 @@
--- src/share/poudriere/bulk.sh.orig 2024-08-11 06:43:13 UTC
+++ src/share/poudriere/bulk.sh
@@ -47,6 +47,8 @@ Options:
-f file. Implies -c for -a.
-c -- Clean all the previously built binary packages and logs.
-F -- Only fetch from original master_site (skip FreeBSD mirrors)
+ -H -- Create a repository where the package filenames contain the
+ short hash of the contents.
-I -- Advanced Interactive mode. Leaves jail running with ports
installed after test.
-i -- Interactive mode. Enter jail for interactive testing and
@@ -101,7 +103,7 @@ fi
usage
fi
-while getopts "ab:B:CcFf:iIj:J:knNO:p:RrSTtvwz:" FLAG; do
+while getopts "ab:B:CcFf:HiIj:J:knNO:p:RrSTtvwz:" FLAG; do
case "${FLAG}" in
a)
ALL=1
@@ -129,6 +131,9 @@ while getopts "ab:B:CcFf:iIj:J:knNO:p:RrSTtvwz:" FLAG;
OPTARG="${SAVED_PWD}/${OPTARG}"
fi
LISTPKGS="${LISTPKGS:+${LISTPKGS} }${OPTARG}"
+ ;;
+ H)
+ PKG_REPO_FLAGS="${PKG_REPO_FLAGS:+${PKG_REPO_FLAGS} }--hash --symlink"
;;
I)
INTERACTIVE_MODE=2

88
ports-mgmt/poudriere-devel/files/patch-src_share_poudriere_common.sh Normal file
View File

@ -0,0 +1,88 @@
--- src/share/poudriere/common.sh.orig 2024-08-11 06:43:13 UTC
+++ src/share/poudriere/common.sh
@@ -6626,6 +6626,10 @@ _delete_old_pkg() {
if [ -L "${pkg}" ]; then
is_sym=1
fi
+ if [ -d "${pkg}" ] && [ "${pkgfile}" = "Hashed" ]; then
+ msg_debug "Ignoring directory"
+ return 0;
+ fi
if [ "${is_sym}" -eq 1 ] && [ ! -e "${pkg}" ]; then
msg "Deleting ${COLOR_PORT}${pkgfile}${COLOR_RESET}: dead symlink"
delete_pkg "${pkg}"
@@ -9835,12 +9839,16 @@ build_repo() {
}
build_repo() {
- local origin pkg_repo_list_files
+ local origin pkg_repo_list_files hashcmd
msg "Creating pkg repository"
if [ ${DRY_RUN} -eq 1 ]; then
return 0
fi
+ if [ ${PKG_HASH} != "no" ]; then
+ hashcmd="--hash --symlink"
+ PKG_REPO_FLAGS="${PKG_REPO_FLAGS:+${PKG_REPO_FLAGS} }$hashcmd"
+ fi
bset status "pkgrepo:"
ensure_pkg_installed force_extract || \
err 1 "Unable to extract pkg."
@@ -9860,12 +9868,20 @@ build_repo() {
install -m 0400 "${PKG_REPO_META_FILE}" \
"${MASTERMNT:?}/tmp/pkgmeta"
fi
+
+ # Remount rw
+ # mount_nullfs does not support mount -u
+ umount ${UMOUNT_NONBUSY} ${MASTERMNT}/packages || \
+ umount -f ${MASTERMNT}/packages
+ mount_packages
+
mkdir -p ${MASTERMNT}/tmp/packages
if [ -n "${PKG_REPO_SIGNING_KEY}" ]; then
msg "Signing repository with key: ${PKG_REPO_SIGNING_KEY}"
install -m 0400 "${PKG_REPO_SIGNING_KEY}" \
"${MASTERMNT:?}/tmp/repo.key"
injail ${PKG_BIN:?} repo \
+ ${PKG_REPO_FLAGS} \
${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
-o /tmp/packages \
${PKG_META} \
@@ -9882,6 +9898,7 @@ build_repo() {
# using SSH with DNSSEC as older hosts don't support
# it.
${MASTERMNT:?}${PKG_BIN:?} repo \
+ ${PKG_REPO_FLAGS} \
${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
-o "${MASTERMNT:?}/tmp/packages" ${PKG_META_MASTERMNT} \
"${MASTERMNT:?}/packages" \
@@ -9894,6 +9911,7 @@ build_repo() {
;;
esac
JNETNAME="n" injail ${PKG_BIN:?} repo \
+ ${PKG_REPO_FLAGS} \
${pkg_repo_list_files:+"${pkg_repo_list_files}"} \
-o /tmp/packages ${PKG_META} /packages \
${SIGNING_COMMAND:+signing_command: ${SIGNING_COMMAND}} ||
@@ -9909,6 +9927,11 @@ build_repo() {
sign_pkg pubkey "${PACKAGES:?}/Latest/pkg.${PKG_EXT}"
fi
fi
+
+ # Remount ro
+ umount ${UMOUNT_NONBUSY} ${MASTERMNT}/packages || \
+ umount -f ${MASTERMNT}/packages
+ mount_packages -o ro
}
calculate_size_in_mb() {
@@ -10477,6 +10500,7 @@ esac
: ${FLAVOR_DEFAULT_ALL:=no}
: ${NULLFS_PATHS:="/rescue /usr/share /usr/tests /usr/lib32"}
: ${PACKAGE_FETCH_URL:="pkg+http://pkg.FreeBSD.org/\${ABI}"}
+: ${PKG_HASH:=no}
: ${POUDRIERE_TMPDIR:=$(command mktemp -dt poudriere)}
: ${SHASH_VAR_PATH_DEFAULT:=${POUDRIERE_TMPDIR}}

22
ports-mgmt/poudriere-devel/files/patch-src_share_poudriere_include_pkg.sh Normal file
View File

@ -0,0 +1,22 @@
--- src/share/poudriere/include/pkg.sh.orig 2024-08-11 06:43:13 UTC
+++ src/share/poudriere/include/pkg.sh
@@ -466,6 +466,10 @@ delete_pkg() {
local pkg="$1"
clear_pkg_cache "${pkg}"
+
+ # If ${pkg} is a symlink, delete the target as well
+ [ -L "${pkg}" ] && unlink $(realpath "${pkg}")
+
# Delete the package and the depsfile since this package is being deleted,
# which will force it to be recreated
unlink "${pkg}"
@@ -483,6 +487,8 @@ delete_pkg_xargs() {
# Delete the package and the depsfile since this package is being deleted,
# which will force it to be recreated
{
+ # If ${pkg} is a symlink, delete the target as well
+ [ -L "${pkg}" ] && echo $(realpath "${pkg}")
echo "${pkg}"
echo "${pkg_cache_dir}"
} >> "${listfile}"

20
ports-mgmt/poudriere-devel/files/patch-src_share_poudriere_pkgclean.sh Normal file
View File

@ -0,0 +1,20 @@
--- src/share/poudriere/pkgclean.sh.orig 2024-08-11 06:43:13 UTC
+++ src/share/poudriere/pkgclean.sh
@@ -289,6 +289,8 @@ check_should_delete_pkg() {
*".${PKG_EXT}")
if should_delete "${file}"; then
echo "${file}" >> "${BADFILES_LIST:?}"
+ # If the pkg is a symlink to a hashed package, remove the hashed version as well
+ [ -L "${file}" ] && echo "$(realpath ${file})" >> ${BADFILES_LIST}
fi
;;
*.txz)
@@ -305,6 +307,8 @@ check_should_delete_pkg() {
*)
msg_verbose "Found incorrect format file: ${file}"
echo "${file}" >> "${BADFILES_LIST:?}"
+ # If the pkg is a symlink to a hashed package, remove the hashed version as well
+ [ -L "${file}" ] && echo "$(realpath ${file})" >> ${BADFILES_LIST}
;;
esac
}