From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.1 (2015-04-28) on sa.local.altlinux.org X-Spam-Level: X-Spam-Status: No, score=-1.8 required=5.0 tests=BAYES_00,DKIM_SIGNED, T_DKIM_INVALID autolearn=no autolearn_force=no version=3.4.1 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=altlinux.org; s=dkim; h=Subject:Content-Transfer-Encoding:MIME-Version: References:In-Reply-To:Message-Id:Date:Cc:To:From:Sender:Reply-To: Content-Type:Content-ID:Content-Description:Resent-Date:Resent-From: Resent-Sender:Resent-To:Resent-Cc:Resent-Message-ID:List-Id:List-Help: List-Unsubscribe:List-Subscribe:List-Post:List-Owner:List-Archive; bh=/RTrNtgXwHzUsSqlJwwtVC3BDm0RoUMHMZ9QkrnfpjA=; b=dm8th5Tp4hw2p+U+6LVVCG44in AoIRnPh5+2+fkK3lFLgi1M7QbJ6ivXjD36A30tY79GvJqnJG6wxVmrk5S4CFDLkYE3sgDFv3X/m+t /MZslVtxc3LJaqGBh/vbkT42PDF6gFJ3Lff3tA3F8AjyeeJjQzMC7rFgDSEpPTtRHDIHszmEtLdnf X/im2zWH6p2z0wg3+HeMppHZ7mIodrQSOqDVpH5R7K3xSPIMIn/2FmHmpUvScNTW/I7XcHDgNVN4c NhYdBrmHf0YcSz4so8pMrDoJCGHq0BhvRVwG8jtae0lZ3/G2ide3cVdnZHnkUSvy9fltWR7cDdZ9q x7HIl3Jg==; From: "Vladimir D. Seleznev" To: devel@lists.altlinux.org Date: Thu, 4 Jun 2020 22:58:11 +0300 Message-Id: <20200604195811.3881130-2-vseleznv@altlinux.org> X-Mailer: git-send-email 2.25.4 In-Reply-To: <20200604195811.3881130-1-vseleznv@altlinux.org> References: <20200604195811.3881130-1-vseleznv@altlinux.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SA-Exim-Connect-IP: 46.39.229.193 X-SA-Exim-Mail-From: vseleznv@altlinux.org X-SA-Exim-Version: 4.2.1 X-SA-Exim-Scanned: Yes (on mail.cs.msu.ru) Cc: vseleznv@altlinux.org Subject: [devel] [PATCH] gb: add gb-task-build-post, optimize packages with identical rebuild X-BeenThere: devel@lists.altlinux.org X-Mailman-Version: 2.1.12 Precedence: list Reply-To: ALT Linux Team development discussions List-Id: ALT Linux Team development discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 04 Jun 2020 19:58:36 -0000 Archived-At: List-Archive: List-Post: Introduce task post-build processing. It finds subtasks with package rebuild and if the rebuilt packages identical to the same packages in the target repo it optimizes them. Whether subtasks in the particular repository should be check or not for such optimization is controlling via GB_OPTIMIZE_IDENTICAL_REBUILD option. * gb/gb-build-task: Add gb-task-build-post. * gb/gb-sh-conf: Add GB_OPTIMIZE_IDENTICAL_REBUILD option. * gb/gb-task-build-post: New file. * gb/gb-task-build-post-i: Likewise. --- gb/gb-build-task | 1 + gb/gb-sh-conf | 1 + gb/gb-task-build-post | 9 ++++ gb/gb-task-build-post-i | 114 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 125 insertions(+) create mode 100755 gb/gb-task-build-post create mode 100755 gb/gb-task-build-post-i diff --git a/gb/gb-build-task b/gb/gb-build-task index 0fb076e..5f4b23e 100755 --- a/gb/gb-build-task +++ b/gb/gb-build-task @@ -61,6 +61,7 @@ gb-task-set-summary gb-task-copy-packages gb-task-build-prep gb-task-build + gb-task-build-post fail_if_task_abort_requested gb-x-girar task-make-index-html "$id" ||: diff --git a/gb/gb-sh-conf b/gb/gb-sh-conf index f1366fe..52bc79e 100644 --- a/gb/gb-sh-conf +++ b/gb/gb-sh-conf @@ -38,3 +38,4 @@ GB_AREPO_DIR="$TMPDIR/gb-arepo-$USER-$GB_REPO_NAME" remote_host="build-$USER-$GB_REPO_NAME-$arch" : ${GB_ALLOW_SAME_NEVR=} +: ${GB_OPTIMIZE_IDENTICAL_REBUILD=} diff --git a/gb/gb-task-build-post b/gb/gb-task-build-post new file mode 100755 index 0000000..e0af1ff --- /dev/null +++ b/gb/gb-task-build-post @@ -0,0 +1,9 @@ +#!/bin/sh -efu + +. gb-sh-functions + +fail_if_task_abort_requested + +for i in $(src_nums); do + $0-i "$i" +done diff --git a/gb/gb-task-build-post-i b/gb/gb-task-build-post-i new file mode 100755 index 0000000..104c294 --- /dev/null +++ b/gb/gb-task-build-post-i @@ -0,0 +1,114 @@ +#!/bin/sh -efu + +. gb-sh-functions +. gb-sh-tmpdir + +[ -n "$GB_OPTIMIZE_IDENTICAL_REBUILD" ] || + exit 0 + +i="$1"; shift +query_filename='%{name}-%{version}-%{release}.%{arch}.rpm' + +is_rebuild= +st_is_rebuild() +{ + [ -z "$is_rebuild" ] || + return 0 + + local arch="$1"; shift + + srpm="$(find "build/$i/$arch/srpm/" \ + -mindepth 1 -maxdepth 1 -type f \ + -name '*.rpm' -printf '%f')" + + [ -f "$GB_REPO_DIR/files/SRPMS/$srpm" ] || + return 1 + + is_rebuild=1 + + return 0 +} + +mixed_arches= +narchs= +get_narchs() +{ + [ -z "$narchs" ] || + return 0 + + narchs="$(find "build/$i/$arch/rpms/" \ + -mindepth 1 -maxdepth 1 \ + -type f -name '*.rpm' \ + -execdir rpmquery --qf '%{arch}\n' -p '{}' '+' | + sort -u |wc -l)" + + [ "$narchs" -eq 1 ] || + mixed_arches=1 +} + +stamp_echo >&2 "#$i: Starting post build processing" + +first_arch=1 +for arch in $GB_ARCH; do + [ ! -s "build/$i/$arch/excluded" ] || + continue + + # exit if subtask is not a rebuild + st_is_rebuild "$arch" || { + stamp_echo >&2 "#$i: subtask is not a rebuild, skipping" + exit 0 + } + + get_narchs + + find "build/$i/$arch/rpms/" -mindepth 1 -maxdepth 1 -type f -name '*.rpm' \ + -execdir rpmquery --qf "$query_filename %{arch}\n" \ + -p '{}' '+' >"$tmpdir/built.$arch.pkgs" + + while read -r brpm barch; do + # debuginfo rpms are out of our interest + case "$brpm" in + *-debuginfo-*) continue ;; + esac + + # skip non-first built noarch rpm in mixed arches subtask + [ -z "$mixed_arches" ] || [ -z "$first_arch" ] || [ "$barch" != "noarch" ] || + continue + + bid="$(rpmidentity -p "build/$i/$arch/rpms/$brpm")" + printf "%s %s %s\n" "$brpm" "$barch" "$bid" >>"$tmpdir/built.identity" + rid="$(rpmidentity -p "$GB_REPO_DIR/files/$barch/RPMS/$brpm")" + printf "%s %s %s\n" "$brpm" "$barch" "$rid" >>"$tmpdir/repo.identity" + done <"$tmpdir/built.$arch.pkgs" + + first_arch= +done + +sort -k 3 -o "$tmpdir/built.identity"{,} +sort -k 3 -o "$tmpdir/repo.identity"{,} +join -j 3 -v2 "$tmpdir/built.identity" "$tmpdir/repo.identity" | + cut -d' ' -f2 >"$tmpdir/nonidentical.arches" + +# Mixed noarch and arches build is a special case: we have to optimize either +# all packages of all architectures or none of them +if [ -n "$mixed_arches" -a -s "$tmpdir/nonidentical.arches" ]; then + echo >&2 "#$i: non-identical rebuild with mixed arches, skip optimizing" + exit 0 +fi + +for arch in $GB_ARCH; do + [ ! -s "build/$i/$arch/excluded" ] || + continue + + if grep -q "$arch" "$tmpdir/nonidentical.arches"; then + echo >&2 "[$arch] #$i rebuild is non-identical, skip optimizing" + continue + fi + + echo >&2 "[$arch] #$i rebuild is identical, optimize packages" + while read brpm barch; do + cp -p "$GB_REPO_DIR/files/$barch/RPMS/$brpm" "build/$i/$arch/rpms/$brpm" + echo >&2 "[$arch] #$i $brpm is optimized" + done <"$tmpdir/built.$arch.pkgs" +done + -- 2.25.4