From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Date: Wed, 17 Jul 2019 22:05:22 +0300 From: Michael Shigorin To: devel-distro@lists.altlinux.org Message-ID: <20190717190522.GA32557@imap.altlinux.org> MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="k1lZvvs/B4yU6o8G" Content-Disposition: inline Content-Transfer-Encoding: 8bit User-Agent: Mutt/1.5.23.88.hg577987ca2d02 (2014-03-12) Subject: [devel-distro] =?koi8-r?b?W20tcF0gZ3JvdXBzOiDB19TP0M/J08sg2sHX?= =?koi8-r?b?ydPJzc/T1MXK?= X-BeenThere: devel-distro@lists.altlinux.org X-Mailman-Version: 2.1.12 Precedence: list Reply-To: Distributions development List-Id: Distributions development List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 17 Jul 2019 19:05:23 -0000 Archived-At: List-Archive: --k1lZvvs/B4yU6o8G Content-Type: text/plain; charset=koi8-r Content-Disposition: inline Content-Transfer-Encoding: 8bit Здравствуйте. Прилагаю набор коммитов, добавляющий в mkimage-profiles автоматическое дополнение указанных явно groups и lists списками, рекурсивно вытащенными из групп, перечисленных в profiles либо заданных явно -- см. profiles2groups с allgroups в image.in/functions.mk и bin/groups2deps, за который отдельное спасибо Диме Левину (у меня уже соображалка засыпала). Это реализовано в коммитах 0002 и 0004. "В нагрузку" прилагаю ещё пару коммитов из master..next, где 0001 переносит возможность указания архитектуры (а-ля @X86) и на группы в profile-файлах; 0003 исправляет порядок копирования profiles, groups, lists на обратный (чтобы работала постобработка насчёт опять же @arch). И есть ещё пара штук помельче, которым давно было место, но не было времени и внимания. PS: что спросить-то хотел: есть ли противопоказания к такому автопоиску? Бывало ли, что надо уложить группы, но не их зависимости (тоже группы)? --  ---- WBR, Michael Shigorin / http://altlinux.org   ------ http://opennet.ru / http://anna-news.info --k1lZvvs/B4yU6o8G Content-Type: text/x-patch; charset=us-ascii Content-Disposition: attachment; filename="0001-pkg.in-factored-out-bin-archdep.patch" >>From 608e11a25a6c44bffe018b4d7a6281fffc32554d Mon Sep 17 00:00:00 2001 From: Michael Shigorin Date: Tue, 16 Jul 2019 22:41:59 +0300 Subject: [PATCH 1/4] pkg.in: factored out bin/archdep-* These were just asking to be unified as a (hasty) internal fork between implementations has actually occured to date (which is a total shame); the overall number of fork()s should go down a bit, too. The archdep-filter is now able to handle both a file specified on the command line and stdin (don't you do inplace substitution on the same file in a pipe though, it's a race condition that occured to be overly subtle to me this evening). Thanks: Dmitry Levin --- bin/archdep-filter | 36 ++++++++++++++++++++++++++++++++++++ bin/archdep-grep | 9 +++++++++ pkg.in/lists/Makefile | 10 +--------- pkg.in/profiles/Makefile | 10 +--------- 4 files changed, 47 insertions(+), 18 deletions(-) create mode 100755 bin/archdep-filter create mode 100755 bin/archdep-grep diff --git a/bin/archdep-filter b/bin/archdep-filter new file mode 100755 index 0000000..3a81b2d --- /dev/null +++ b/bin/archdep-filter @@ -0,0 +1,36 @@ +#!/bin/sh +# transform texts including lines like entity@arch + +[ -n "$ARCH" ] || { + echo "** error: ARCH not set" >&2 + exit 1 +} + +# -n is definitely not a typo +process() { + sed -r -e 's/\<([^@ ]*)@X86\>/\1@i586 \1@x86_64/g' \ + -e 's/\<([^@ ]*)@IA32\>/\1@i586 i586-\1@x86_64/g' \ + -e 's/\<([^@ ]*)@E2K\>/\1@e2k \1@e2kv4/g' | + sed -r -n 's/\<([^@ ]*)\>|\<([^@ ]*)@'"$ARCH"'\>/\1\2/pg' | + sed -r -e 's/\<([^@ ]*)@[^@ ]+\> *//g' \ + -e 's/^ +//;s/ +$//' +} + +exit_handler() +{ + local rc=$1 + rm -f -- "$tmpfile" + exit $rc +} + +tmpfile="$(mktemp)" +trap 'exit_handler $?' EXIT HUP PIPE INT QUIT TERM + +# arguments can be a single file or none +# (in which case stdout gets processed) +if [ -f "$1" ]; then + process < "$1" > "$tmpfile" && + cat "$tmpfile" > "$1" +else + process +fi diff --git a/bin/archdep-grep b/bin/archdep-grep new file mode 100755 index 0000000..a3c3b9d --- /dev/null +++ b/bin/archdep-grep @@ -0,0 +1,9 @@ +#!/bin/sh +# used twice at least + +[ -d "$1" ] || { + echo "** error: missing directory argument" >&2 + exit 1 +} + +find "$1" -type f | xargs -rL1 -I '__' archdep-filter __ diff --git a/pkg.in/lists/Makefile b/pkg.in/lists/Makefile index 1be3792..afc5035 100644 --- a/pkg.in/lists/Makefile +++ b/pkg.in/lists/Makefile @@ -35,16 +35,8 @@ copy-groups: @echo $(call groups2lists) \ | xargs -r -- cp --parents -at $(TARGET) -- -grep-archdep: a = $(ARCH) grep-archdep: - @# xargs -L 1024 -n 1024 ? -- breaks -I - @find $(TARGET) -type f \ - | xargs -I '__' sh -c '\ - sed -ri "s/\\<([^@ ]*)@X86\\>/\\1@i586 \\1@x86_64/g" __; \ - sed -ri "s/\\<([^@ ]*)@IA32\\>/\\1@i586 i586-\\1@x86_64/g" __;\ - sed -rni "s/\\<([^@ ]*)\\>|\\<([^@ ]*)@$a\\>/\\1\\2/pg" __; \ - sed -ri "s/\\<([^@ ]*)@[^@ ]+\\> *//g" __; \ - sed -ri "s/^ +//;s/ +$$//" __' + @archdep-grep $(TARGET) # do beforehand as foreach gets expanded before recipe execution $(TARGET): diff --git a/pkg.in/profiles/Makefile b/pkg.in/profiles/Makefile index 5251436..056a754 100644 --- a/pkg.in/profiles/Makefile +++ b/pkg.in/profiles/Makefile @@ -18,16 +18,8 @@ copy-profiles: $(addsuffix .directory,$(THE_PROFILES)); \ fi -### exact copy of a snippet from ../lists/Makefile -grep-archdep: a = $(ARCH) grep-archdep: - @# xargs -L 1024 -n 1024 ? -- breaks -I - @find $(TARGET) -type f \ - | xargs -I '__' sh -c '\ - sed -ri "s/\\<([^@ ]*)@X86\\>/\\1@i586 \\1@x86_64/g" __; \ - sed -ri "s/\\<([^@ ]*)@IA32\\>/\\1@i586 i586-\\1@x86_64/g" __;\ - sed -rni "s/\\<([^@ ]*)\\>|\\<([^@ ]*)@$a\\>/\\1\\2/pg" __; \ - sed -ri "s/\\<([^@ ]*)@[^@ ]+\\> *//g" __' + @archdep-grep $(TARGET) $(TARGET): @mkdir -p $(TARGET) -- 2.10.4 --k1lZvvs/B4yU6o8G Content-Type: text/x-patch; charset=us-ascii Content-Disposition: attachment; filename="0002-image.in-functions.mk-automatic-profiles2groups.patch" >>From df44cf35a5e5c539a8406d4dd5b57823f26a84be Mon Sep 17 00:00:00 2001 From: Michael Shigorin Date: Tue, 16 Jul 2019 22:49:47 +0300 Subject: [PATCH 2/4] image.in/functions.mk: automatic profiles2groups It dawned on me this evening that there was no such thing as automatic group (and then list) names extraction from pkg.in/profiles in m-p; and I sort of assumed there was one implemented already. Here it is, actually being _the_ reason to finally factor out that archdep-filter script. --- image.in/functions.mk | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/image.in/functions.mk b/image.in/functions.mk index 17e4a3a..548a6a6 100644 --- a/image.in/functions.mk +++ b/image.in/functions.mk @@ -20,12 +20,26 @@ profile = $(addprefix $(PKGDIR)/,$(call rprofile,$(1))) # map first argument (a function) onto second one (an argument list) map = $(foreach a,$(2),$(call $(1),$(a))) -# happens at least twice, and variables are the same by design +# needed once, split out for readability +profiles2groups = $(shell $(profiles2groups_body)) +define profiles2groups_body +{ if [ -n "$(THE_PROFILES)" ]; then \ + sed -rn 's,^X-Alterator-Groups=(.*)$$,\1,p' \ + $(call map,profile,$(THE_PROFILES)) \ + /dev/null \ + | archdep-filter; \ +fi; } +endef + +# happens at least thrice, and variables are the same by design groups2lists = $(shell $(groups2lists_body)) define groups2lists_body -{ if [ -n "$(THE_GROUPS)$(MAIN_GROUPS)" ]; then \ - sed -rn 's,^X-Alterator-PackageList=(.*)$$,\1,p' \ - $(call map,group,$(THE_GROUPS) $(MAIN_GROUPS)); \ +{ if [ -n "$(THE_GROUPS)$(MAIN_GROUPS)$(THE_PROFILES)" ]; then \ + sed -rn 's,^X-Alterator-PackageList=(.*)$$,\1,p' \ + $(call map,group, \ + $(THE_GROUPS) $(MAIN_GROUPS) \ + $(call profiles2groups)) \ + /dev/null; \ fi; } endef -- 2.10.4 --k1lZvvs/B4yU6o8G Content-Type: text/x-patch; charset=us-ascii Content-Disposition: attachment; filename="0003-pkg.in-Makefile-fix-subdir-order.patch" >>From 72e7810938d7c158471cf3c5889420f8b2571872 Mon Sep 17 00:00:00 2001 From: Michael Shigorin Date: Tue, 16 Jul 2019 22:53:43 +0300 Subject: [PATCH 3/4] pkg.in/Makefile: fix subdir order The old one was totally broken: we processed lower level targets before the high-level ones, and the waterfall logic is exactly the opposite here: profiles may yield groups, and groups may yield lists. This is important as lists copied are fixed up with archdep postprocessing, and if higher-level targets add up more lists afterwards these would end up as-is. --- pkg.in/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg.in/Makefile b/pkg.in/Makefile index 2afdae1..e875725 100644 --- a/pkg.in/Makefile +++ b/pkg.in/Makefile @@ -8,6 +8,6 @@ endif # sequential execution, no reason to haste all: - @for dir in lists groups profiles; do \ + @for dir in profiles groups lists; do \ $(MAKE) -C $$dir || exit; \ done -- 2.10.4 --k1lZvvs/B4yU6o8G Content-Type: text/x-patch; charset=us-ascii Content-Disposition: attachment; filename="0004-image.in-pkg.in-groups-inter-group-dependencies.patch" >>From 76b28d72cf7da2a460c1e92dd39b4a5105537b8c Mon Sep 17 00:00:00 2001 From: Michael Shigorin Date: Wed, 17 Jul 2019 21:15:06 +0300 Subject: [PATCH 4/4] image.in, pkg.in/groups: inter-group dependencies The whole affair with automatic extraction hasn't ended with profiles2groups and groups2lists functions as we've also got dependencies between package groups -- both up and down; these can be recursive, and so far nothing stops them from being circular as well. The typical problems that should be solved by this commit are: - missing parent groups (with presumed installer list items turning into toplevel ones); - missing dependency groups (leaving warnings during image build and silent errors during installation IIRC). groups2deps script's core logic was written by Dmitry Levin , I was too sleepy to write down the invariant handling. NB: someone might prefer to disable this kind of service some day; drop me a note (or a knob patch) if you're that someone. --- bin/groups2deps | 32 ++++++++++++++++++++++++++++++++ image.in/functions.mk | 19 ++++++++++++------- pkg.in/groups/Makefile | 9 ++++++++- 3 files changed, 52 insertions(+), 8 deletions(-) create mode 100755 bin/groups2deps diff --git a/bin/groups2deps b/bin/groups2deps new file mode 100755 index 0000000..1e42639 --- /dev/null +++ b/bin/groups2deps @@ -0,0 +1,32 @@ +#!/bin/sh +# extract dependencies from directory files +# corresponding to specified group names +# as a sorted group name list + +exit_handler() +{ + local rc=$1 + rm -f -- "$list" "$next" + exit $rc +} + +list="$(mktemp)" +next="$(mktemp)" + +trap 'exit_handler $?' EXIT HUP PIPE INT QUIT TERM + +# convert to group file names +g2f() { sed -r 's,\<[^ ]+\>,groups/&.directory,g'; } + +# extract dependencies +g2d() { sed -rn 's,^X-Alterator-(Depends|Parent)=(.*)$,\2,p' "$@" /dev/null; } + +if [ -n "$*" ]; then echo "$@"; else cat; fi | tr ' ' '\n' | sort -u > "$list" + +while :; do + g2d $(g2f < "$list") | sort -u | comm -23 - "$list" > "$next" + [ -s "$next" ] || break + sort -u -o "$list"{,} "$next" +done + +cat "$list" diff --git a/image.in/functions.mk b/image.in/functions.mk index 548a6a6..c9975a7 100644 --- a/image.in/functions.mk +++ b/image.in/functions.mk @@ -31,16 +31,21 @@ define profiles2groups_body fi; } endef +# collect groups recursively +# NB: no cyclic graph protection +allgroups = $(shell $(allgroups_body)) +define allgroups_body +{ if [ -n "$(THE_GROUPS)$(MAIN_GROUPS)$(THE_PROFILES)" ]; then \ + cd $(PKGDIR) && \ + groups2deps $(THE_GROUPS) $(MAIN_GROUPS) $(profiles2groups); \ +fi; } +endef + # happens at least thrice, and variables are the same by design groups2lists = $(shell $(groups2lists_body)) define groups2lists_body -{ if [ -n "$(THE_GROUPS)$(MAIN_GROUPS)$(THE_PROFILES)" ]; then \ - sed -rn 's,^X-Alterator-PackageList=(.*)$$,\1,p' \ - $(call map,group, \ - $(THE_GROUPS) $(MAIN_GROUPS) \ - $(call profiles2groups)) \ - /dev/null; \ -fi; } +{ sed -rn 's,^X-Alterator-PackageList=(.*)$$,\1,p' \ + $(call map,group,$(allgroups)) /dev/null; } endef # kernel package list generation; see also #24669 diff --git a/pkg.in/groups/Makefile b/pkg.in/groups/Makefile index 5d79727..802d54f 100644 --- a/pkg.in/groups/Makefile +++ b/pkg.in/groups/Makefile @@ -33,19 +33,26 @@ check-pkglists: else include $(BUILDDIR)/distcfg.mk +include $(BUILDDIR)/functions.mk + SUFFIX := pkg/groups TARGET := $(BUILDDIR)/$(SUFFIX) +all: PKGDIR = .. all: $(GLOBAL_DEBUG) @if [ -n "$(THE_GROUPS)$(MAIN_GROUPS)" ]; then \ mkdir -p $(TARGET) && \ cp --parents -at $(TARGET) -- \ - $(addsuffix .directory,$(THE_GROUPS) $(MAIN_GROUPS)); \ + $(addsuffix .directory, \ + $(THE_GROUPS) $(MAIN_GROUPS) $(allgroups)); \ mp-commit "$(TARGET)" "requested $(SUFFIX) copied over"; \ fi debug: @echo "** THE_GROUPS: $(THE_GROUPS)" @echo "** MAIN_GROUPS: $(MAIN_GROUPS)" +ifeq (2,$(DEBUG)) + @echo "** allgroups: $(call allgroups)" +endif endif -- 2.10.4 --k1lZvvs/B4yU6o8G--