From fc5d57a9a4fe3e3662c4ec4e84772dcea64979ca Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 23 Dec 2022 01:25:31 +0900 Subject: kbuild: refactor silent mode detection Factor out $(findstring s,...). Signed-off-by: Masahiro Yamada Reviewed-by: Nick Desaulniers Reviewed-by: Nicolas Schier --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index c1ead4cd2342a..e2dbaab2f802d 100644 --- a/Makefile +++ b/Makefile @@ -100,12 +100,12 @@ endif # make-4.0 (and later) keep single letter options in the 1st word of MAKEFLAGS. ifeq ($(filter 3.%,$(MAKE_VERSION)),) -silence:=$(findstring s,$(firstword -$(MAKEFLAGS))) +short-opts := $(firstword -$(MAKEFLAGS)) else -silence:=$(findstring s,$(filter-out --%,$(MAKEFLAGS))) +short-opts := $(filter-out --%,$(MAKEFLAGS)) endif -ifeq ($(silence),s) +ifneq ($(findstring s,$(short-opts)),) quiet=silent_ KBUILD_VERBOSE = 0 endif -- cgit 1.2.3-korg From 8962b6b475bddc011c414f40ffd02f0ed4e02771 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 23 Dec 2022 01:25:32 +0900 Subject: kbuild: print short log in addition to the whole command with V=1 "make V=1" prints the whole command instead of the short log, but I think it is nicer to print both so that you can easily spot the build rule of your interest. This commit changes V=1 to print the short log (the line starts with '#'), followed by the full log. In parallel builds, the short/full logs from the same build rule may be interspersed. If you want to avoid it, please add -Otarget option. Kbuild will never set it by default because Make would buffer the logs and lose the escape sequences. (Modern compilers print warnings and errors in color, but only when they write to a terminal.) This is also a preparation for supporting V=12 because V=2 appends the reason for rebuilding to the short log. Signed-off-by: Masahiro Yamada Tested-by: Nicolas Schier Reviewed-by: Nicolas Schier --- Makefile | 18 +++++++----------- scripts/Kbuild.include | 26 +++++++++++++------------- 2 files changed, 20 insertions(+), 24 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index e2dbaab2f802d..9a919428a726d 100644 --- a/Makefile +++ b/Makefile @@ -56,22 +56,18 @@ unexport GREP_OPTIONS # Beautify output # --------------------------------------------------------------------------- # -# Normally, we echo the whole command before executing it. By making -# that echo $($(quiet)$(cmd)), we now have the possibility to set -# $(quiet) to choose other forms of output instead, e.g. +# Most of build commands in Kbuild start with "cmd_". You can optionally define +# "quiet_cmd_*". If defined, the short log is printed. Otherwise, no log from +# that command is printed by default. # -# quiet_cmd_cc_o_c = Compiling $(RELDIR)/$@ -# cmd_cc_o_c = $(CC) $(c_flags) -c -o $@ $< -# -# If $(quiet) is empty, the whole command will be printed. -# If it is set to "quiet_", only the short version will be printed. -# If it is set to "silent_", nothing will be printed at all, since -# the variable $(silent_cmd_cc_o_c) doesn't exist. +# e.g.) +# quiet_cmd_depmod = DEPMOD $(MODLIB) +# cmd_depmod = $(srctree)/scripts/depmod.sh $(DEPMOD) $(KERNELRELEASE) # # A simple variant is to prefix commands with $(Q) - that's useful # for commands that shall be hidden in non-verbose mode. # -# $(Q)ln $@ :< +# $(Q)$(MAKE) $(build)=scripts/basic # # If KBUILD_VERBOSE equals 0 then the above command will be hidden. # If KBUILD_VERBOSE equals 1 then the above command is displayed. diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index 2f7356b2990bb..faae3c7986b07 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -125,15 +125,15 @@ dtbinst := -f $(srctree)/scripts/Makefile.dtbinst obj # $(Q)$(MAKE) $(clean)=dir clean := -f $(srctree)/scripts/Makefile.clean obj -# echo command. -# Short version is used, if $(quiet) equals `quiet_', otherwise full one. -echo-cmd = $(if $($(quiet)cmd_$(1)),\ - echo ' $(call escsq,$($(quiet)cmd_$(1)))$(echo-why)';) - -# sink stdout for 'make -s' - redirect := - quiet_redirect := -silent_redirect := exec >/dev/null; +# pring log +# +# If quiet is "silent_", print nothing and sink stdout +# If quiet is "quiet_", print short log +# If quiet is empty, print short log and whole command +silent_log_print = exec >/dev/null; + quiet_log_print = $(if $(quiet_cmd_$1), echo ' $(call escsq,$(quiet_cmd_$1)$(why))';) + log_print = echo '$(pound) $(call escsq,$(or $(quiet_cmd_$1),cmd_$1 $@))'; \ + echo ' $(call escsq,$(cmd_$1))'; # Delete the target on interruption # @@ -156,8 +156,8 @@ delete-on-interrupt = \ $(foreach sig, HUP INT QUIT TERM PIPE, \ trap 'rm -f $@; trap - $(sig); kill -s $(sig) $$$$' $(sig);)) -# printing commands -cmd = @set -e; $(echo-cmd) $($(quiet)redirect) $(delete-on-interrupt) $(cmd_$(1)) +# print and execute commands +cmd = @$(if $(cmd_$(1)),set -e; $($(quiet)log_print) $(delete-on-interrupt) $(cmd_$(1)),:) ### # if_changed - execute command if any prerequisite is newer than @@ -234,7 +234,7 @@ if_changed_rule = $(if $(if-changed-cond),$(rule_$(1)),@:) # (6) No dir/.target.cmd file and target not listed in $(targets) # This is a good hint that there is a bug in the kbuild file ifeq ($(KBUILD_VERBOSE),2) -why = \ +_why = \ $(if $(filter $@, $(PHONY)),- due to target is PHONY, \ $(if $(wildcard $@), \ $(if $(newer-prereqs),- due to: $(newer-prereqs), \ @@ -251,7 +251,7 @@ why = \ ) \ ) -echo-why = $(call escsq, $(strip $(why))) +why = $(space)$(strip $(_why)) endif ############################################################################### -- cgit 1.2.3-korg From 6ae4b9868a8f723cae2600722eea033fafadd399 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 23 Dec 2022 01:25:34 +0900 Subject: kbuild: allow to combine multiple V= levels Commit a6de553da01c ("kbuild: Allow to combine multiple W= levels") supported W=123 to enable all the extra warning groups. I think a similar idea is applicable to the V= option. V=1 echos the whole command V=2 prints the reason for rebuilding These are orthogonal, and can be enabled at the same time. This commit supports V=12 to enable both of them. Signed-off-by: Masahiro Yamada Tested-by: Nicolas Schier Reviewed-by: Nicolas Schier --- Makefile | 18 +++++++++--------- arch/x86/tools/Makefile | 2 +- scripts/Kbuild.include | 4 ++-- scripts/tags.sh | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 9a919428a726d..ed5ce58196be1 100644 --- a/Makefile +++ b/Makefile @@ -69,9 +69,8 @@ unexport GREP_OPTIONS # # $(Q)$(MAKE) $(build)=scripts/basic # -# If KBUILD_VERBOSE equals 0 then the above command will be hidden. -# If KBUILD_VERBOSE equals 1 then the above command is displayed. -# If KBUILD_VERBOSE equals 2 then give the reason why each target is rebuilt. +# If KBUILD_VERBOSE contains 1, the whole command is echoed. +# If KBUILD_VERBOSE contains 2, the reason for rebuilding is printed. # # To put more focus on warnings, be less verbose as default # Use 'make V=1' to see the full commands @@ -83,12 +82,12 @@ ifndef KBUILD_VERBOSE KBUILD_VERBOSE = 0 endif -ifeq ($(KBUILD_VERBOSE),1) +quiet = quiet_ +Q = @ + +ifneq ($(findstring 1, $(KBUILD_VERBOSE)),) quiet = Q = -else - quiet=quiet_ - Q = @ endif # If the user is running make -s (silent mode), suppress echoing of @@ -1775,8 +1774,9 @@ help: printf " %-16s - Show all of the above\\n" help-boards; \ echo '') - @echo ' make V=0|1 [targets] 0 => quiet build (default), 1 => verbose build' - @echo ' make V=2 [targets] 2 => give reason for rebuild of target' + @echo ' make V=n [targets] 0: quiet build (default), 1: verbose build' + @echo ' 2: give reason for rebuild of target' + @echo ' V=1 and V=2 can be combined with V=12' @echo ' make O=dir [targets] Locate all output files in "dir", including .config' @echo ' make C=1 [targets] Check re-compiled c source with $$CHECK' @echo ' (sparse by default)' diff --git a/arch/x86/tools/Makefile b/arch/x86/tools/Makefile index bddfc9a466453..90e820ac9771f 100644 --- a/arch/x86/tools/Makefile +++ b/arch/x86/tools/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 PHONY += posttest -ifeq ($(KBUILD_VERBOSE),1) +ifneq ($(findstring 1, $(KBUILD_VERBOSE)),) posttest_verbose = -v else posttest_verbose = diff --git a/scripts/Kbuild.include b/scripts/Kbuild.include index faae3c7986b07..10cf8d2d82ef4 100644 --- a/scripts/Kbuild.include +++ b/scripts/Kbuild.include @@ -132,7 +132,7 @@ clean := -f $(srctree)/scripts/Makefile.clean obj # If quiet is empty, print short log and whole command silent_log_print = exec >/dev/null; quiet_log_print = $(if $(quiet_cmd_$1), echo ' $(call escsq,$(quiet_cmd_$1)$(why))';) - log_print = echo '$(pound) $(call escsq,$(or $(quiet_cmd_$1),cmd_$1 $@))'; \ + log_print = echo '$(pound) $(call escsq,$(or $(quiet_cmd_$1),cmd_$1 $@)$(why))'; \ echo ' $(call escsq,$(cmd_$1))'; # Delete the target on interruption @@ -233,7 +233,7 @@ if_changed_rule = $(if $(if-changed-cond),$(rule_$(1)),@:) # (5) No dir/.target.cmd file (used to store command line) # (6) No dir/.target.cmd file and target not listed in $(targets) # This is a good hint that there is a bug in the kbuild file -ifeq ($(KBUILD_VERBOSE),2) +ifneq ($(findstring 2, $(KBUILD_VERBOSE)),) _why = \ $(if $(filter $@, $(PHONY)),- due to target is PHONY, \ $(if $(wildcard $@), \ diff --git a/scripts/tags.sh b/scripts/tags.sh index e137cf15aae9d..2e756bee1fa95 100755 --- a/scripts/tags.sh +++ b/scripts/tags.sh @@ -8,7 +8,7 @@ # Uses the following environment variables: # SUBARCH, SRCARCH, srctree -if [ "$KBUILD_VERBOSE" = "1" ]; then +if [[ "$KBUILD_VERBOSE" =~ 1 ]]; then set -x fi -- cgit 1.2.3-korg From 83d98d73b4fc2d485b4a2fd996f5a23e79ee3b52 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 23 Dec 2022 01:25:35 +0900 Subject: kbuild: drop V=0 support The top Makefile sets KBUILD_VERBOSE to 0 by default, it looks weird now because V=1 and V=2 can be OR'ed as V=12. The default should be empty. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- Documentation/kbuild/makefiles.rst | 4 ++-- Makefile | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) (limited to 'Makefile') diff --git a/Documentation/kbuild/makefiles.rst b/Documentation/kbuild/makefiles.rst index 38bc74eaa5474..82363218d0d19 100644 --- a/Documentation/kbuild/makefiles.rst +++ b/Documentation/kbuild/makefiles.rst @@ -523,7 +523,7 @@ more details, with real examples. $(Q)$(MAKE) $(build)=$(boot) MACHINE=$(MACHINE) $(boot)/$@ @$(kecho) ' Kernel: $(boot)/$@ is ready' - When kbuild is executing with KBUILD_VERBOSE=0, then only a shorthand + When kbuild is executing with KBUILD_VERBOSE unset, then only a shorthand of a command is normally displayed. To enable this behaviour for custom commands kbuild requires two variables to be set:: @@ -544,7 +544,7 @@ more details, with real examples. GEN lib/crc32table.h - will be displayed with "make KBUILD_VERBOSE=0". + will be displayed with "make KBUILD_VERBOSE=". 3.12 Command change detection ----------------------------- diff --git a/Makefile b/Makefile index ed5ce58196be1..cb5f433cf2004 100644 --- a/Makefile +++ b/Makefile @@ -78,9 +78,6 @@ unexport GREP_OPTIONS ifeq ("$(origin V)", "command line") KBUILD_VERBOSE = $(V) endif -ifndef KBUILD_VERBOSE - KBUILD_VERBOSE = 0 -endif quiet = quiet_ Q = @ @@ -102,7 +99,7 @@ endif ifneq ($(findstring s,$(short-opts)),) quiet=silent_ -KBUILD_VERBOSE = 0 +override KBUILD_VERBOSE := endif export quiet Q KBUILD_VERBOSE @@ -1774,7 +1771,7 @@ help: printf " %-16s - Show all of the above\\n" help-boards; \ echo '') - @echo ' make V=n [targets] 0: quiet build (default), 1: verbose build' + @echo ' make V=n [targets] 1: verbose build' @echo ' 2: give reason for rebuild of target' @echo ' V=1 and V=2 can be combined with V=12' @echo ' make O=dir [targets] Locate all output files in "dir", including .config' -- cgit 1.2.3-korg From 91ecf7ff1b036f3fe1183809661119b1ee109b19 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 29 Dec 2022 16:43:10 +0900 Subject: kbuild: make W=1 warn files that are tracked but ignored by git The top .gitignore comments about how to detect files breaking .gitignore rules, but people rarely care about it. Add a new W=1 warning to detect files that are tracked but ignored by git. If git is not installed or the source tree is not tracked by git at all, this script does not print anything. Running it on v6.2-rc1 detected the following: $ make W=1 misc-check Documentation/devicetree/bindings/.yamllint: warning: ignored by one of the .gitignore files drivers/clk/.kunitconfig: warning: ignored by one of the .gitignore files drivers/gpu/drm/tests/.kunitconfig: warning: ignored by one of the .gitignore files drivers/hid/.kunitconfig: warning: ignored by one of the .gitignore files fs/ext4/.kunitconfig: warning: ignored by one of the .gitignore files fs/fat/.kunitconfig: warning: ignored by one of the .gitignore files kernel/kcsan/.kunitconfig: warning: ignored by one of the .gitignore files lib/kunit/.kunitconfig: warning: ignored by one of the .gitignore files mm/kfence/.kunitconfig: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/.gitignore: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/Makefile: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/run_tags_test.sh: warning: ignored by one of the .gitignore files tools/testing/selftests/arm64/tags/tags_test.c: warning: ignored by one of the .gitignore files These are ignored by the '.*' or 'tags' in the top .gitignore, but there is no rule to negate it. You might be tempted to do 'git add -f' but I want to have the real issue fixed (by fixing a .gitignore, or by renaming files, etc.). Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor Reviewed-by: Nicolas Schier --- Makefile | 6 ++++++ scripts/misc-check | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100755 scripts/misc-check (limited to 'Makefile') diff --git a/Makefile b/Makefile index cb5f433cf2004..b9d1411ef25ba 100644 --- a/Makefile +++ b/Makefile @@ -1861,6 +1861,12 @@ rust-analyzer: # Misc # --------------------------------------------------------------------------- +PHONY += misc-check +misc-check: + $(Q)$(srctree)/scripts/misc-check + +all: misc-check + PHONY += scripts_gdb scripts_gdb: prepare0 $(Q)$(MAKE) $(build)=scripts/gdb diff --git a/scripts/misc-check b/scripts/misc-check new file mode 100755 index 0000000000000..d40d5484e0c50 --- /dev/null +++ b/scripts/misc-check @@ -0,0 +1,19 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-only + +set -e + +# Detect files that are tracked but ignored by git. This is checked only when +# ${KBUILD_EXTRA_WARN} contains 1, git is installed, and the source tree is +# tracked by git. +check_tracked_ignored_files () { + case "${KBUILD_EXTRA_WARN}" in + *1*) ;; + *) return;; + esac + + git -C ${srctree:-.} ls-files -i -c --exclude-per-directory=.gitignore 2>/dev/null | + sed 's/$/: warning: ignored by one of the .gitignore files/' >&2 +} + +check_tracked_ignored_files -- cgit 1.2.3-korg From c83b16cefd78f55071840e1159ead9fe62747769 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 7 Jan 2023 18:45:45 +0900 Subject: kbuild: rust: move rust/target.json to scripts/ scripts/ is a better place to generate files used treewide. With target.json moved to scripts/, you do not need to add target.json to no-clean-files or MRPROPER_FILES. 'make clean' does not visit scripts/, but 'make mrproper' does. Signed-off-by: Masahiro Yamada Reviewed-by: Miguel Ojeda Tested-by: Miguel Ojeda --- Makefile | 4 ++-- rust/.gitignore | 1 - rust/Makefile | 10 +--------- scripts/.gitignore | 1 + scripts/Makefile | 8 +++++++- scripts/remove-stale-files | 2 ++ 6 files changed, 13 insertions(+), 13 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index b9d1411ef25ba..ef6d593634cf5 100644 --- a/Makefile +++ b/Makefile @@ -569,7 +569,7 @@ KBUILD_CFLAGS := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \ -std=gnu11 KBUILD_CPPFLAGS := -D__KERNEL__ KBUILD_RUSTFLAGS := $(rust_common_flags) \ - --target=$(objtree)/rust/target.json \ + --target=$(objtree)/scripts/target.json \ -Cpanic=abort -Cembed-bitcode=n -Clto=n \ -Cforce-unwind-tables=n -Ccodegen-units=1 \ -Csymbol-mangling-version=v0 \ @@ -1606,7 +1606,7 @@ MRPROPER_FILES += include/config include/generated \ certs/x509.genkey \ vmlinux-gdb.py \ *.spec \ - rust/target.json rust/libmacros.so + rust/libmacros.so # clean - Delete most, but leave enough to build external modules # diff --git a/rust/.gitignore b/rust/.gitignore index 9bd1af8e05a16..168cb26a31b99 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -1,6 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -target.json bindings_generated.rs bindings_helpers_generated.rs exports_*_generated.h diff --git a/rust/Makefile b/rust/Makefile index f403b79cae5a0..829f4adfd85cf 100644 --- a/rust/Makefile +++ b/rust/Makefile @@ -1,8 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 -always-$(CONFIG_RUST) += target.json -no-clean-files += target.json - obj-$(CONFIG_RUST) += core.o compiler_builtins.o always-$(CONFIG_RUST) += exports_core_generated.h @@ -231,11 +228,6 @@ rusttest-kernel: $(src)/kernel/lib.rs rusttest-prepare \ $(call if_changed,rustc_test) $(call if_changed,rustc_test_library) -filechk_rust_target = $(objtree)/scripts/generate_rust_target < $< - -$(obj)/target.json: $(objtree)/include/config/auto.conf FORCE - $(call filechk,rust_target) - ifdef CONFIG_CC_IS_CLANG bindgen_c_flags = $(c_flags) else @@ -359,7 +351,7 @@ rust-analyzer: $(obj)/core.o: private skip_clippy = 1 $(obj)/core.o: private skip_flags = -Dunreachable_pub $(obj)/core.o: private rustc_target_flags = $(core-cfgs) -$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs $(obj)/target.json FORCE +$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs scripts/target.json FORCE $(call if_changed_dep,rustc_library) $(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*' diff --git a/scripts/.gitignore b/scripts/.gitignore index b7aec8eb1bd44..11bf3c075fb67 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -8,4 +8,5 @@ /recordmcount /sign-file /sorttable +/target.json /unifdef diff --git a/scripts/Makefile b/scripts/Makefile index 1575af84d557b..0e0ae3c06ed74 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -10,8 +10,14 @@ hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT) += sorttable hostprogs-always-$(CONFIG_ASN1) += asn1_compiler hostprogs-always-$(CONFIG_MODULE_SIG_FORMAT) += sign-file hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE) += insert-sys-cert -hostprogs-always-$(CONFIG_RUST) += generate_rust_target +always-$(CONFIG_RUST) += target.json +filechk_rust_target = $< < include/config/auto.conf + +$(obj)/target.json: scripts/generate_rust_target include/config/auto.conf FORCE + $(call filechk,rust_target) + +hostprogs += generate_rust_target generate_rust_target-rust := y HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include diff --git a/scripts/remove-stale-files b/scripts/remove-stale-files index cdbdde89a2711..c71bf2f683603 100755 --- a/scripts/remove-stale-files +++ b/scripts/remove-stale-files @@ -27,3 +27,5 @@ rm -f arch/x86/purgatory/kexec-purgatory.c rm -f scripts/extract-cert rm -f scripts/kconfig/[gmnq]conf-cfg + +rm -f rust/target.json -- cgit 1.2.3-korg From 9c73bcfaa4308c2f2c4871110deb1bc089931942 Mon Sep 17 00:00:00 2001 From: Thomas Weißschuh Date: Wed, 18 Jan 2023 05:05:34 +0000 Subject: kbuild: also delete temporary directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reuse the standard naming schema for temporary files also for temporary directories. Such a directory will be used by the kheaders generation. Signed-off-by: Thomas Weißschuh Reviewed-by: Nicolas Schier Tested-by: Nicolas Schier Signed-off-by: Masahiro Yamada --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index ef6d593634cf5..05cb0ec3ea941 100644 --- a/Makefile +++ b/Makefile @@ -2038,11 +2038,12 @@ clean: $(clean-dirs) -o -name '*.lex.c' -o -name '*.tab.[ch]' \ -o -name '*.asn1.[ch]' \ -o -name '*.symtypes' -o -name 'modules.order' \ - -o -name '.tmp_*' \ -o -name '*.c.[012]*.*' \ -o -name '*.ll' \ -o -name '*.gcno' \ - -o -name '*.*.symversions' \) -type f -print | xargs rm -f + -o -name '*.*.symversions' \) -type f -print \ + -o -name '.tmp_*' -print \ + | xargs rm -rf # Generate tags for editors # --------------------------------------------------------------------------- -- cgit 1.2.3-korg From 8d9acfce33329d1f4b0f0969a9ba884bea7501c6 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 11 Jan 2023 20:05:11 -0700 Subject: kbuild: Stop using '-Qunused-arguments' with clang This option masks all unused command line argument warnings, which can hide potential issues, such as an architecture Makefile adding an unsupported flag to KBUILD_AFLAGS or KBUILD_CFLAGS, which will cause all as-option and cc-options to silently fail due to -Werror with no indication as to why in the main kernel build. Remove this flag so that warnings of this nature can be caught early and obviously in a build. Signed-off-by: Nathan Chancellor Reviewed-by: Nick Desaulniers Tested-by: Linux Kernel Functional Testing Tested-by: Anders Roxell Signed-off-by: Masahiro Yamada --- Makefile | 1 - 1 file changed, 1 deletion(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 05cb0ec3ea941..cf30a8b6463e5 100644 --- a/Makefile +++ b/Makefile @@ -870,7 +870,6 @@ KBUILD_RUSTFLAGS-$(CONFIG_WERROR) += -Dwarnings KBUILD_RUSTFLAGS += $(KBUILD_RUSTFLAGS-y) ifdef CONFIG_CC_IS_CLANG -KBUILD_CPPFLAGS += -Qunused-arguments # The kernel builds with '-std=gnu11' so use of GNU extensions is acceptable. KBUILD_CFLAGS += -Wno-gnu else -- cgit 1.2.3-korg From ec31f868ec674edfcf653cc7c82b365c6f570cd9 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 22 Jan 2023 23:14:25 +0900 Subject: setlocalversion: absorb $(KERNELVERSION) Print $(KERNELVERSION) in setlocalversion so that the callers get simpler. Signed-off-by: Masahiro Yamada --- Makefile | 5 ++--- scripts/setlocalversion | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index cf30a8b6463e5..203b86e951976 100644 --- a/Makefile +++ b/Makefile @@ -1258,8 +1258,7 @@ vmlinux: vmlinux.o $(KBUILD_LDS) modpost # make sure no implicit rule kicks in $(sort $(KBUILD_LDS) $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS)): . ; -filechk_kernel.release = \ - echo "$(KERNELVERSION)$$($(CONFIG_SHELL) $(srctree)/scripts/setlocalversion $(srctree))" +filechk_kernel.release = $(srctree)/scripts/setlocalversion $(srctree) # Store (new) KERNELRELEASE string in include/config/kernel.release include/config/kernel.release: FORCE @@ -2124,7 +2123,7 @@ checkstack: $(PERL) $(srctree)/scripts/checkstack.pl $(CHECKSTACK_ARCH) kernelrelease: - @echo "$(KERNELVERSION)$$($(CONFIG_SHELL) $(srctree)/scripts/setlocalversion $(srctree))" + @$(srctree)/scripts/setlocalversion $(srctree) kernelversion: @echo $(KERNELVERSION) diff --git a/scripts/setlocalversion b/scripts/setlocalversion index b8e1018d611e5..ad72ea2c82288 100755 --- a/scripts/setlocalversion +++ b/scripts/setlocalversion @@ -102,6 +102,11 @@ if ! test -e include/config/auto.conf; then exit 1 fi +if [ -z "${KERNELVERSION}" ]; then + echo "KERNELVERSION is not set" >&2 + exit 1 +fi + # localversion* files in the build and source directory res="$(collect_files localversion*)" if test ! "$srctree" -ef .; then @@ -127,4 +132,4 @@ elif [ "${LOCALVERSION+set}" != "set" ]; then res="$res$(scm_version --short)" fi -echo "$res" +echo "${KERNELVERSION}${res}" -- cgit 1.2.3-korg From 1cb86b6c313623486038165f90f4067578c2f5d5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 28 Jan 2023 01:19:42 +0900 Subject: kbuild: save overridden KERNELRELEASE in include/config/kernel.release ${KERNELRELEASE} is used as a part of the installation path. (INSTALL_DTBS_PATH, MODLIB, etc.) When KERNELRELEASE is overridden from the command line, it should be saved in include/config/kernel.release, so that it will be consistently used for the installation steps. Signed-off-by: Masahiro Yamada --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 203b86e951976..3e7f96f12f089 100644 --- a/Makefile +++ b/Makefile @@ -1258,7 +1258,11 @@ vmlinux: vmlinux.o $(KBUILD_LDS) modpost # make sure no implicit rule kicks in $(sort $(KBUILD_LDS) $(KBUILD_VMLINUX_OBJS) $(KBUILD_VMLINUX_LIBS)): . ; +ifeq ($(origin KERNELRELEASE),file) filechk_kernel.release = $(srctree)/scripts/setlocalversion $(srctree) +else +filechk_kernel.release = echo $(KERNELRELEASE) +endif # Store (new) KERNELRELEASE string in include/config/kernel.release include/config/kernel.release: FORCE @@ -2123,7 +2127,7 @@ checkstack: $(PERL) $(srctree)/scripts/checkstack.pl $(CHECKSTACK_ARCH) kernelrelease: - @$(srctree)/scripts/setlocalversion $(srctree) + @$(filechk_kernel.release) kernelversion: @echo $(KERNELVERSION) -- cgit 1.2.3-korg From 4e3feaad6ff8a7a57e3bf3308a93c93e3a2e17a6 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 24 Jan 2023 09:19:28 -0700 Subject: powerpc/vdso: Filter clang's auto var init zero enabler when linking After commit 8d9acfce3332 ("kbuild: Stop using '-Qunused-arguments' with clang"), the PowerPC vDSO shows the following error with clang-13 and older when CONFIG_INIT_STACK_ALL_ZERO is enabled: clang: error: argument unused during compilation: '-enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang' [-Werror,-Wunused-command-line-argument] clang-14 added a change to make sure this flag never triggers -Wunused-command-line-argument, so it is fixed with newer releases. For older releases that the kernel still supports building with, just filter out this flag, as has been done for other flags. Fixes: f0a42fbab447 ("powerpc/vdso: Improve linker flags") Fixes: 8d9acfce3332 ("kbuild: Stop using '-Qunused-arguments' with clang") Link: https://github.com/llvm/llvm-project/commit/ca6d5813d17598cd180995fb3bdfca00f364475f Signed-off-by: Nathan Chancellor Signed-off-by: Masahiro Yamada --- Makefile | 4 +++- arch/powerpc/kernel/vdso/Makefile | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 3e7f96f12f089..5bd587bb30b57 100644 --- a/Makefile +++ b/Makefile @@ -912,7 +912,9 @@ ifdef CONFIG_INIT_STACK_ALL_ZERO KBUILD_CFLAGS += -ftrivial-auto-var-init=zero ifdef CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO_ENABLER # https://github.com/llvm/llvm-project/issues/44842 -KBUILD_CFLAGS += -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang +CC_AUTO_VAR_INIT_ZERO_ENABLER := -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang +export CC_AUTO_VAR_INIT_ZERO_ENABLER +KBUILD_CFLAGS += $(CC_AUTO_VAR_INIT_ZERO_ENABLER) endif endif diff --git a/arch/powerpc/kernel/vdso/Makefile b/arch/powerpc/kernel/vdso/Makefile index 7e69e87fbf744..d2a1dc0956d51 100644 --- a/arch/powerpc/kernel/vdso/Makefile +++ b/arch/powerpc/kernel/vdso/Makefile @@ -56,7 +56,7 @@ ccflags-y := -fno-common -fno-builtin ldflags-y := -Wl,--hash-style=both -nostdlib -shared -z noexecstack ldflags-$(CONFIG_LD_IS_LLD) += $(call cc-option,--ld-path=$(LD),-fuse-ld=lld) # Filter flags that clang will warn are unused for linking -ldflags-y += $(filter-out $(CC_FLAGS_FTRACE) -Wa$(comma)%, $(KBUILD_CFLAGS)) +ldflags-y += $(filter-out $(CC_AUTO_VAR_INIT_ZERO_ENABLER) $(CC_FLAGS_FTRACE) -Wa$(comma)%, $(KBUILD_CFLAGS)) CC32FLAGS := -m32 LD32FLAGS := -Wl,-soname=linux-vdso32.so.1 -- cgit 1.2.3-korg From feb113ad8be1bafcd0a3b93bae639be939af563c Mon Sep 17 00:00:00 2001 From: Carlos Llamas Date: Wed, 25 Jan 2023 18:30:47 +0000 Subject: kbuild: fix trivial typo in comment Add missing underscore in CONFIG_DEBUG_INFO_BTF_MODULES. Fixes: f73edc8951b2 ("kbuild: unify two modpost invocations") Signed-off-by: Carlos Llamas Signed-off-by: Masahiro Yamada --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 5bd587bb30b57..93796ece1cba4 100644 --- a/Makefile +++ b/Makefile @@ -1538,7 +1538,7 @@ endif # Build modules # -# *.ko are usually independent of vmlinux, but CONFIG_DEBUG_INFOBTF_MODULES +# *.ko are usually independent of vmlinux, but CONFIG_DEBUG_INFO_BTF_MODULES # is an exception. ifdef CONFIG_DEBUG_INFO_BTF_MODULES KBUILD_BUILTIN := 1 -- cgit 1.2.3-korg From 67d7c3023a672c2b73d19d6d23684df670fce648 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sat, 28 Jan 2023 18:24:23 +0900 Subject: kbuild: remove --include-dir MAKEFLAG from top Makefile I added $(srctree)/ to some included Makefiles in the following commits: - 3204a7fb98a3 ("kbuild: prefix $(srctree)/ to some included Makefiles") - d82856395505 ("kbuild: do not require sub-make for separate output tree builds") They were a preparation for removing --include-dir flag. I have never thought --include-dir useful. Rather, it _is_ harmful. For example, run the following commands: $ make -s ARCH=x86 mrproper defconfig $ make ARCH=arm O=foo dtbs make[1]: Entering directory '/tmp/linux/foo' HOSTCC scripts/basic/fixdep Error: kernelrelease not valid - run 'make prepare' to update it UPD include/config/kernel.release make[1]: Leaving directory '/tmp/linux/foo' The first command configures the source tree for x86. The next command tries to build ARM device trees in the separate foo/ directory - this must stop because the directory foo/ has not been configured yet. However, due to --include-dir=$(abs_srctree), the top Makefile includes the wrong include/config/auto.conf from the source tree and continues building. Kbuild traverses the directory tree, but of course it does not work correctly. The Error message is also pointless - 'make prepare' does not help at all for fixing the issue. This commit fixes more arch Makefile, and finally removes --include-dir from the top Makefile. There are more breakages under drivers/, but I do not volunteer to fix them all. I just moved --include-dir to drivers/Makefile. With this commit, the second command will stop with a sensible message. $ make -s ARCH=x86 mrproper defconfig $ make ARCH=arm O=foo dtbs make[1]: Entering directory '/tmp/linux/foo' SYNC include/config/auto.conf.cmd *** *** The source tree is not clean, please run 'make ARCH=arm mrproper' *** in /tmp/linux *** make[2]: *** [../Makefile:646: outputmakefile] Error 1 /tmp/linux/Makefile:770: include/config/auto.conf.cmd: No such file or directory make[1]: *** [/tmp/linux/Makefile:793: include/config/auto.conf.cmd] Error 2 make[1]: Leaving directory '/tmp/linux/foo' make: *** [Makefile:226: __sub-make] Error 2 Signed-off-by: Masahiro Yamada --- Makefile | 8 -------- arch/arm/mach-s3c/Makefile | 4 ++-- arch/ia64/kernel/Makefile | 2 +- arch/mips/Kbuild | 2 +- arch/mips/Makefile.postlink | 2 +- arch/powerpc/Makefile.postlink | 2 +- arch/um/drivers/Makefile | 2 +- arch/um/kernel/Makefile | 2 +- arch/um/kernel/skas/Makefile | 2 +- arch/um/os-Linux/Makefile | 2 +- arch/um/os-Linux/drivers/Makefile | 2 +- arch/um/os-Linux/skas/Makefile | 2 +- arch/x86/Makefile.um | 2 +- arch/x86/um/Makefile | 2 +- arch/x86/um/os-Linux/Makefile | 2 +- drivers/Makefile | 5 +++++ fs/hostfs/Makefile | 2 +- 17 files changed, 21 insertions(+), 24 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 93796ece1cba4..2faf872b68089 100644 --- a/Makefile +++ b/Makefile @@ -203,14 +203,6 @@ ifneq ($(words $(subst :, ,$(abs_srctree))), 1) $(error source directory cannot contain spaces or colons) endif -ifneq ($(abs_srctree),$(abs_objtree)) -# Look for make include files relative to root of kernel src -# -# --included-dir is added for backward compatibility, but you should not rely on -# it. Please add $(srctree)/ prefix to include Makefiles in the source tree. -MAKEFLAGS += --include-dir=$(abs_srctree) -endif - ifneq ($(filter 3.%,$(MAKE_VERSION)),) # 'MAKEFLAGS += -rR' does not immediately become effective for GNU Make 3.x # We need to invoke sub-make to avoid implicit rules in the top Makefile. diff --git a/arch/arm/mach-s3c/Makefile b/arch/arm/mach-s3c/Makefile index 7c7d3318fd613..deb44326b8286 100644 --- a/arch/arm/mach-s3c/Makefile +++ b/arch/arm/mach-s3c/Makefile @@ -3,11 +3,11 @@ # Copyright 2009 Simtec Electronics ifdef CONFIG_ARCH_S3C24XX -include $(src)/Makefile.s3c24xx +include $(srctree)/$(src)/Makefile.s3c24xx endif ifdef CONFIG_ARCH_S3C64XX -include $(src)/Makefile.s3c64xx +include $(srctree)/$(src)/Makefile.s3c64xx endif # Objects we always build independent of SoC choice diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile index ae9ff07de4abe..d7e1cabee2ec4 100644 --- a/arch/ia64/kernel/Makefile +++ b/arch/ia64/kernel/Makefile @@ -43,4 +43,4 @@ obj-$(CONFIG_ELF_CORE) += elfcore.o CFLAGS_traps.o += -mfixed-range=f2-f5,f16-f31 # The gate DSO image is built using a special linker script. -include $(src)/Makefile.gate +include $(srctree)/$(src)/Makefile.gate diff --git a/arch/mips/Kbuild b/arch/mips/Kbuild index 9e8071f0e58ff..af2967bffb73d 100644 --- a/arch/mips/Kbuild +++ b/arch/mips/Kbuild @@ -7,7 +7,7 @@ subdir-ccflags-y := -Werror endif # platform specific definitions -include arch/mips/Kbuild.platforms +include $(srctree)/arch/mips/Kbuild.platforms obj-y := $(platform-y) # make clean traverses $(obj-) without having included .config, so diff --git a/arch/mips/Makefile.postlink b/arch/mips/Makefile.postlink index 4b1d3ba3a8a28..34e3bd71f3b0e 100644 --- a/arch/mips/Makefile.postlink +++ b/arch/mips/Makefile.postlink @@ -10,7 +10,7 @@ PHONY := __archpost __archpost: -include include/config/auto.conf -include scripts/Kbuild.include +include $(srctree)/scripts/Kbuild.include CMD_LS3_LLSC = arch/mips/tools/loongson3-llsc-check quiet_cmd_ls3_llsc = LLSCCHK $@ diff --git a/arch/powerpc/Makefile.postlink b/arch/powerpc/Makefile.postlink index a6c77f4d32b27..1f860b3c9bec5 100644 --- a/arch/powerpc/Makefile.postlink +++ b/arch/powerpc/Makefile.postlink @@ -9,7 +9,7 @@ PHONY := __archpost __archpost: -include include/config/auto.conf -include scripts/Kbuild.include +include $(srctree)/scripts/Kbuild.include quiet_cmd_head_check = CHKHEAD $@ cmd_head_check = $(CONFIG_SHELL) $(srctree)/arch/powerpc/tools/head_check.sh "$(NM)" "$@" diff --git a/arch/um/drivers/Makefile b/arch/um/drivers/Makefile index e1dc4292bd22e..dee6f66353b33 100644 --- a/arch/um/drivers/Makefile +++ b/arch/um/drivers/Makefile @@ -72,4 +72,4 @@ CFLAGS_null.o = -DDEV_NULL=$(DEV_NULL_PATH) CFLAGS_xterm.o += '-DCONFIG_XTERM_CHAN_DEFAULT_EMULATOR="$(CONFIG_XTERM_CHAN_DEFAULT_EMULATOR)"' -include arch/um/scripts/Makefile.rules +include $(srctree)/arch/um/scripts/Makefile.rules diff --git a/arch/um/kernel/Makefile b/arch/um/kernel/Makefile index 1c2d4b29a3d46..811188be954ca 100644 --- a/arch/um/kernel/Makefile +++ b/arch/um/kernel/Makefile @@ -29,7 +29,7 @@ obj-$(CONFIG_GENERIC_PCI_IOMAP) += ioport.o USER_OBJS := config.o -include arch/um/scripts/Makefile.rules +include $(srctree)/arch/um/scripts/Makefile.rules targets := config.c config.tmp capflags.c diff --git a/arch/um/kernel/skas/Makefile b/arch/um/kernel/skas/Makefile index f3d494a4fd9ba..f93972a257653 100644 --- a/arch/um/kernel/skas/Makefile +++ b/arch/um/kernel/skas/Makefile @@ -14,4 +14,4 @@ UNPROFILE_OBJS := clone.o KCOV_INSTRUMENT := n -include arch/um/scripts/Makefile.rules +include $(srctree)/arch/um/scripts/Makefile.rules diff --git a/arch/um/os-Linux/Makefile b/arch/um/os-Linux/Makefile index 77ac50baa3f82..544e0b344c754 100644 --- a/arch/um/os-Linux/Makefile +++ b/arch/um/os-Linux/Makefile @@ -18,4 +18,4 @@ USER_OBJS := $(user-objs-y) elf_aux.o execvp.o file.o helper.o irq.o \ main.o mem.o process.o registers.o sigio.o signal.o start_up.o time.o \ tty.o umid.o util.o -include arch/um/scripts/Makefile.rules +include $(srctree)/arch/um/scripts/Makefile.rules diff --git a/arch/um/os-Linux/drivers/Makefile b/arch/um/os-Linux/drivers/Makefile index d79e75f1b69a2..cf2d75bb18848 100644 --- a/arch/um/os-Linux/drivers/Makefile +++ b/arch/um/os-Linux/drivers/Makefile @@ -10,4 +10,4 @@ obj-y = obj-$(CONFIG_UML_NET_ETHERTAP) += ethertap.o obj-$(CONFIG_UML_NET_TUNTAP) += tuntap.o -include arch/um/scripts/Makefile.rules +include $(srctree)/arch/um/scripts/Makefile.rules diff --git a/arch/um/os-Linux/skas/Makefile b/arch/um/os-Linux/skas/Makefile index c4566e7888150..75f11989d2e90 100644 --- a/arch/um/os-Linux/skas/Makefile +++ b/arch/um/os-Linux/skas/Makefile @@ -7,4 +7,4 @@ obj-y := mem.o process.o USER_OBJS := $(obj-y) -include arch/um/scripts/Makefile.rules +include $(srctree)/arch/um/scripts/Makefile.rules diff --git a/arch/x86/Makefile.um b/arch/x86/Makefile.um index b3c1ae084180d..b89e2e0024c57 100644 --- a/arch/x86/Makefile.um +++ b/arch/x86/Makefile.um @@ -17,7 +17,7 @@ LDS_EXTRA := -Ui386 export LDS_EXTRA # First of all, tune CFLAGS for the specific CPU. This actually sets cflags-y. -include arch/x86/Makefile_32.cpu +include $(srctree)/arch/x86/Makefile_32.cpu # prevent gcc from keeping the stack 16 byte aligned. Taken from i386. cflags-y += $(call cc-option,-mpreferred-stack-boundary=2) diff --git a/arch/x86/um/Makefile b/arch/x86/um/Makefile index 3d5cd2e578202..ee89f6bb92426 100644 --- a/arch/x86/um/Makefile +++ b/arch/x86/um/Makefile @@ -48,4 +48,4 @@ include/generated/user_constants.h: $(obj)/user-offsets.s FORCE UNPROFILE_OBJS := stub_segv.o CFLAGS_stub_segv.o := $(CFLAGS_NO_HARDENING) -include arch/um/scripts/Makefile.rules +include $(srctree)/arch/um/scripts/Makefile.rules diff --git a/arch/x86/um/os-Linux/Makefile b/arch/x86/um/os-Linux/Makefile index 253bfb8cb702d..ae169125d03fc 100644 --- a/arch/x86/um/os-Linux/Makefile +++ b/arch/x86/um/os-Linux/Makefile @@ -10,4 +10,4 @@ obj-$(CONFIG_64BIT) += prctl.o USER_OBJS := $(obj-y) -include arch/um/scripts/Makefile.rules +include $(srctree)/arch/um/scripts/Makefile.rules diff --git a/drivers/Makefile b/drivers/Makefile index bdf1c66141c9b..62a9cb403a1a1 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -6,6 +6,11 @@ # Rewritten to use lists instead of if-statements. # +# Some driver Makefiles miss $(srctree)/ for include directive. +ifdef building_out_of_srctree +MAKEFLAGS += --include-dir=$(srctree) +endif + obj-y += irqchip/ obj-y += bus/ diff --git a/fs/hostfs/Makefile b/fs/hostfs/Makefile index d5beaffad43b7..587bcd6e50a3a 100644 --- a/fs/hostfs/Makefile +++ b/fs/hostfs/Makefile @@ -8,4 +8,4 @@ hostfs-objs := hostfs_kern.o hostfs_user.o obj-y := obj-$(CONFIG_HOSTFS) += hostfs.o -include arch/um/scripts/Makefile.rules +include $(srctree)/arch/um/scripts/Makefile.rules -- cgit 1.2.3-korg From 5c3d1d0abb12a6915d0f43233837053945621a89 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Wed, 15 Feb 2023 10:20:23 +0900 Subject: kbuild: add a tool to list files ignored by git In short, the motivation of this commit is to build a source package without cleaning the source tree. The deb-pkg and (src)rpm-pkg targets first run 'make clean' before creating a source tarball. Otherwise build artifacts such as *.o, *.a, etc. would be included in the tarball. Yet, the tarball ends up containing several garbage files since 'make clean' does not clean everything. Cleaning the tree every time is annoying since it makes the incremental build impossible. It is desirable to create a source tarball without cleaning the tree. In fact, there are some ways to achieve this. The easiest solution is 'git archive'. 'make perf-tar*-src-pkg' uses it, but I do not like it because it works only when the source tree is managed by git, and all files you want in the tarball must be committed in advance. I want to make it work without relying on git. We can do this. Files that are ignored by git are generated files, so should be excluded from the source tarball. We can list them out by parsing the .gitignore files. Of course, .gitignore does not cover all the cases, but it works well enough. tar(1) claims to support it: --exclude-vcs-ignores Exclude files that match patterns read from VCS-specific ignore files. Supported files are: .cvsignore, .gitignore, .bzrignore, and .hgignore. The best scenario would be to use 'tar --exclude-vcs-ignores', but this option does not work. --exclude-vcs-ignore does not understand any of the negation (!), preceding slash, following slash, etc.. So, this option is just useless. Hence, I wrote this gitignore parser. The previous version [1], written in Python, was so slow. This version is implemented in C, so it works much faster. I imported the code from git (commit: 23c56f7bd5f1), so we get the same result. This tool traverses the source tree, parsing all .gitignore files, and prints file paths that are ignored by git. The output is similar to 'git ls-files --ignored --directory --others --exclude-per-directory=.gitignore', except [1] Not sorted [2] No trailing slash for directories [2] is intentional because tar's --exclude-from option cannot handle trailing slashes. [How to test this tool] $ git clean -dfx $ make -s -j$(nproc) defconfig all # or allmodconifg or whatever $ git archive -o ../linux1.tar --prefix=./ HEAD $ tar tf ../linux1.tar | LANG=C sort > ../file-list1 # files emitted by 'git archive' $ make scripts_package HOSTCC scripts/list-gitignored $ scripts/list-gitignored --prefix=./ -o ../exclude-list $ tar cf ../linux2.tar --exclude-from=../exclude-list . $ tar tf ../linux2.tar | LANG=C sort > ../file-list2 # files emitted by 'tar' $ diff ../file-list1 ../file-list2 | grep -E '^(<|>)' < ./Documentation/devicetree/bindings/.yamllint < ./drivers/clk/.kunitconfig < ./drivers/gpu/drm/tests/.kunitconfig < ./drivers/hid/.kunitconfig < ./fs/ext4/.kunitconfig < ./fs/fat/.kunitconfig < ./kernel/kcsan/.kunitconfig < ./lib/kunit/.kunitconfig < ./mm/kfence/.kunitconfig < ./tools/testing/selftests/arm64/tags/ < ./tools/testing/selftests/arm64/tags/.gitignore < ./tools/testing/selftests/arm64/tags/Makefile < ./tools/testing/selftests/arm64/tags/run_tags_test.sh < ./tools/testing/selftests/arm64/tags/tags_test.c < ./tools/testing/selftests/kvm/.gitignore < ./tools/testing/selftests/kvm/Makefile < ./tools/testing/selftests/kvm/config < ./tools/testing/selftests/kvm/settings The source tarball contains most of files that are tracked by git. You see some diffs, but it is just because some .gitignore files are wrong. $ git ls-files -i -c --exclude-per-directory=.gitignore Documentation/devicetree/bindings/.yamllint drivers/clk/.kunitconfig drivers/gpu/drm/tests/.kunitconfig drivers/hid/.kunitconfig fs/ext4/.kunitconfig fs/fat/.kunitconfig kernel/kcsan/.kunitconfig lib/kunit/.kunitconfig mm/kfence/.kunitconfig tools/testing/selftests/arm64/tags/.gitignore tools/testing/selftests/arm64/tags/Makefile tools/testing/selftests/arm64/tags/run_tags_test.sh tools/testing/selftests/arm64/tags/tags_test.c tools/testing/selftests/kvm/.gitignore tools/testing/selftests/kvm/Makefile tools/testing/selftests/kvm/config tools/testing/selftests/kvm/settings [1]: https://lore.kernel.org/all/20230128173843.765212-1-masahiroy@kernel.org/ Signed-off-by: Masahiro Yamada --- Makefile | 7 +- scripts/.gitignore | 1 + scripts/Makefile | 2 +- scripts/list-gitignored.c | 1057 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1065 insertions(+), 2 deletions(-) create mode 100644 scripts/list-gitignored.c (limited to 'Makefile') diff --git a/Makefile b/Makefile index 2faf872b68089..7c726fd265402 100644 --- a/Makefile +++ b/Makefile @@ -274,7 +274,8 @@ no-dot-config-targets := $(clean-targets) \ cscope gtags TAGS tags help% %docs check% coccicheck \ $(version_h) headers headers_% archheaders archscripts \ %asm-generic kernelversion %src-pkg dt_binding_check \ - outputmakefile rustavailable rustfmt rustfmtcheck + outputmakefile rustavailable rustfmt rustfmtcheck \ + scripts_package # Installation targets should not require compiler. Unfortunately, vdso_install # is an exception where build artifacts may be updated. This must be fixed. no-compiler-targets := $(no-dot-config-targets) install dtbs_install \ @@ -1652,6 +1653,10 @@ distclean: mrproper %pkg: include/config/kernel.release FORCE $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.package $@ +PHONY += scripts_package +scripts_package: scripts_basic + $(Q)$(MAKE) $(build)=scripts scripts/list-gitignored + # Brief documentation of the typical targets used # --------------------------------------------------------------------------- diff --git a/scripts/.gitignore b/scripts/.gitignore index 6e9ce6720a05a..feb43045d1b1e 100644 --- a/scripts/.gitignore +++ b/scripts/.gitignore @@ -3,6 +3,7 @@ /generate_rust_target /insert-sys-cert /kallsyms +/list-gitignored /module.lds /recordmcount /sign-file diff --git a/scripts/Makefile b/scripts/Makefile index 32b6ba7227284..e8917975905ca 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -38,7 +38,7 @@ HOSTCFLAGS_sorttable.o += -DMCOUNT_SORT_ENABLED endif # The following programs are only built on demand -hostprogs += unifdef +hostprogs += list-gitignored unifdef # The module linker script is preprocessed on demand targets += module.lds diff --git a/scripts/list-gitignored.c b/scripts/list-gitignored.c new file mode 100644 index 0000000000000..f9941f8dcd2b3 --- /dev/null +++ b/scripts/list-gitignored.c @@ -0,0 +1,1057 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// Traverse the source tree, parsing all .gitignore files, and print file paths +// that are ignored by git. +// The output is suitable to the --exclude-from option of tar. +// This is useful until the --exclude-vcs-ignores option gets working correctly. +// +// Copyright (C) 2023 Masahiro Yamada +// (a lot of code imported from GIT) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Imported from commit 23c56f7bd5f1667f8b793d796bf30e39545920f6 in GIT +// +//---------------------------(IMPORT FROM GIT BEGIN)--------------------------- + +// Copied from environment.c + +static bool ignore_case; + +// Copied from git-compat-util.h + +/* Sane ctype - no locale, and works with signed chars */ +#undef isascii +#undef isspace +#undef isdigit +#undef isalpha +#undef isalnum +#undef isprint +#undef islower +#undef isupper +#undef tolower +#undef toupper +#undef iscntrl +#undef ispunct +#undef isxdigit + +static const unsigned char sane_ctype[256]; +#define GIT_SPACE 0x01 +#define GIT_DIGIT 0x02 +#define GIT_ALPHA 0x04 +#define GIT_GLOB_SPECIAL 0x08 +#define GIT_REGEX_SPECIAL 0x10 +#define GIT_PATHSPEC_MAGIC 0x20 +#define GIT_CNTRL 0x40 +#define GIT_PUNCT 0x80 +#define sane_istest(x,mask) ((sane_ctype[(unsigned char)(x)] & (mask)) != 0) +#define isascii(x) (((x) & ~0x7f) == 0) +#define isspace(x) sane_istest(x,GIT_SPACE) +#define isdigit(x) sane_istest(x,GIT_DIGIT) +#define isalpha(x) sane_istest(x,GIT_ALPHA) +#define isalnum(x) sane_istest(x,GIT_ALPHA | GIT_DIGIT) +#define isprint(x) ((x) >= 0x20 && (x) <= 0x7e) +#define islower(x) sane_iscase(x, 1) +#define isupper(x) sane_iscase(x, 0) +#define is_glob_special(x) sane_istest(x,GIT_GLOB_SPECIAL) +#define iscntrl(x) (sane_istest(x,GIT_CNTRL)) +#define ispunct(x) sane_istest(x, GIT_PUNCT | GIT_REGEX_SPECIAL | \ + GIT_GLOB_SPECIAL | GIT_PATHSPEC_MAGIC) +#define isxdigit(x) (hexval_table[(unsigned char)(x)] != -1) +#define tolower(x) sane_case((unsigned char)(x), 0x20) +#define toupper(x) sane_case((unsigned char)(x), 0) + +static inline int sane_case(int x, int high) +{ + if (sane_istest(x, GIT_ALPHA)) + x = (x & ~0x20) | high; + return x; +} + +static inline int sane_iscase(int x, int is_lower) +{ + if (!sane_istest(x, GIT_ALPHA)) + return 0; + + if (is_lower) + return (x & 0x20) != 0; + else + return (x & 0x20) == 0; +} + +// Copied from ctype.c + +enum { + S = GIT_SPACE, + A = GIT_ALPHA, + D = GIT_DIGIT, + G = GIT_GLOB_SPECIAL, /* *, ?, [, \\ */ + R = GIT_REGEX_SPECIAL, /* $, (, ), +, ., ^, {, | */ + P = GIT_PATHSPEC_MAGIC, /* other non-alnum, except for ] and } */ + X = GIT_CNTRL, + U = GIT_PUNCT, + Z = GIT_CNTRL | GIT_SPACE +}; + +static const unsigned char sane_ctype[256] = { + X, X, X, X, X, X, X, X, X, Z, Z, X, X, Z, X, X, /* 0.. 15 */ + X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 16.. 31 */ + S, P, P, P, R, P, P, P, R, R, G, R, P, P, R, P, /* 32.. 47 */ + D, D, D, D, D, D, D, D, D, D, P, P, P, P, P, G, /* 48.. 63 */ + P, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, /* 64.. 79 */ + A, A, A, A, A, A, A, A, A, A, A, G, G, U, R, P, /* 80.. 95 */ + P, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, /* 96..111 */ + A, A, A, A, A, A, A, A, A, A, A, R, R, U, P, X, /* 112..127 */ + /* Nothing in the 128.. range */ +}; + +// Copied from hex.c + +static const signed char hexval_table[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, /* 00-07 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 08-0f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 10-17 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 18-1f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 20-27 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 28-2f */ + 0, 1, 2, 3, 4, 5, 6, 7, /* 30-37 */ + 8, 9, -1, -1, -1, -1, -1, -1, /* 38-3f */ + -1, 10, 11, 12, 13, 14, 15, -1, /* 40-47 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 48-4f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 50-57 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 58-5f */ + -1, 10, 11, 12, 13, 14, 15, -1, /* 60-67 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 68-67 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 70-77 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 78-7f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 80-87 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 88-8f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 90-97 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* 98-9f */ + -1, -1, -1, -1, -1, -1, -1, -1, /* a0-a7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* a8-af */ + -1, -1, -1, -1, -1, -1, -1, -1, /* b0-b7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* b8-bf */ + -1, -1, -1, -1, -1, -1, -1, -1, /* c0-c7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* c8-cf */ + -1, -1, -1, -1, -1, -1, -1, -1, /* d0-d7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* d8-df */ + -1, -1, -1, -1, -1, -1, -1, -1, /* e0-e7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* e8-ef */ + -1, -1, -1, -1, -1, -1, -1, -1, /* f0-f7 */ + -1, -1, -1, -1, -1, -1, -1, -1, /* f8-ff */ +}; + +// Copied from wildmatch.h + +#define WM_CASEFOLD 1 +#define WM_PATHNAME 2 + +#define WM_NOMATCH 1 +#define WM_MATCH 0 +#define WM_ABORT_ALL -1 +#define WM_ABORT_TO_STARSTAR -2 + +// Copied from wildmatch.c + +typedef unsigned char uchar; + +// local modification: remove NEGATE_CLASS(2) + +#define CC_EQ(class, len, litmatch) ((len) == sizeof (litmatch)-1 \ + && *(class) == *(litmatch) \ + && strncmp((char*)class, litmatch, len) == 0) + +// local modification: simpilify macros +#define ISBLANK(c) ((c) == ' ' || (c) == '\t') +#define ISGRAPH(c) (isprint(c) && !isspace(c)) +#define ISPRINT(c) isprint(c) +#define ISDIGIT(c) isdigit(c) +#define ISALNUM(c) isalnum(c) +#define ISALPHA(c) isalpha(c) +#define ISCNTRL(c) iscntrl(c) +#define ISLOWER(c) islower(c) +#define ISPUNCT(c) ispunct(c) +#define ISSPACE(c) isspace(c) +#define ISUPPER(c) isupper(c) +#define ISXDIGIT(c) isxdigit(c) + +/* Match pattern "p" against "text" */ +static int dowild(const uchar *p, const uchar *text, unsigned int flags) +{ + uchar p_ch; + const uchar *pattern = p; + + for ( ; (p_ch = *p) != '\0'; text++, p++) { + int matched, match_slash, negated; + uchar t_ch, prev_ch; + if ((t_ch = *text) == '\0' && p_ch != '*') + return WM_ABORT_ALL; + if ((flags & WM_CASEFOLD) && ISUPPER(t_ch)) + t_ch = tolower(t_ch); + if ((flags & WM_CASEFOLD) && ISUPPER(p_ch)) + p_ch = tolower(p_ch); + switch (p_ch) { + case '\\': + /* Literal match with following character. Note that the test + * in "default" handles the p[1] == '\0' failure case. */ + p_ch = *++p; + /* FALLTHROUGH */ + default: + if (t_ch != p_ch) + return WM_NOMATCH; + continue; + case '?': + /* Match anything but '/'. */ + if ((flags & WM_PATHNAME) && t_ch == '/') + return WM_NOMATCH; + continue; + case '*': + if (*++p == '*') { + const uchar *prev_p = p - 2; + while (*++p == '*') {} + if (!(flags & WM_PATHNAME)) + /* without WM_PATHNAME, '*' == '**' */ + match_slash = 1; + else if ((prev_p < pattern || *prev_p == '/') && + (*p == '\0' || *p == '/' || + (p[0] == '\\' && p[1] == '/'))) { + /* + * Assuming we already match 'foo/' and are at + * , just assume it matches + * nothing and go ahead match the rest of the + * pattern with the remaining string. This + * helps make foo/<*><*>/bar (<> because + * otherwise it breaks C comment syntax) match + * both foo/bar and foo/a/bar. + */ + if (p[0] == '/' && + dowild(p + 1, text, flags) == WM_MATCH) + return WM_MATCH; + match_slash = 1; + } else /* WM_PATHNAME is set */ + match_slash = 0; + } else + /* without WM_PATHNAME, '*' == '**' */ + match_slash = flags & WM_PATHNAME ? 0 : 1; + if (*p == '\0') { + /* Trailing "**" matches everything. Trailing "*" matches + * only if there are no more slash characters. */ + if (!match_slash) { + if (strchr((char *)text, '/')) + return WM_NOMATCH; + } + return WM_MATCH; + } else if (!match_slash && *p == '/') { + /* + * _one_ asterisk followed by a slash + * with WM_PATHNAME matches the next + * directory + */ + const char *slash = strchr((char*)text, '/'); + if (!slash) + return WM_NOMATCH; + text = (const uchar*)slash; + /* the slash is consumed by the top-level for loop */ + break; + } + while (1) { + if (t_ch == '\0') + break; + /* + * Try to advance faster when an asterisk is + * followed by a literal. We know in this case + * that the string before the literal + * must belong to "*". + * If match_slash is false, do not look past + * the first slash as it cannot belong to '*'. + */ + if (!is_glob_special(*p)) { + p_ch = *p; + if ((flags & WM_CASEFOLD) && ISUPPER(p_ch)) + p_ch = tolower(p_ch); + while ((t_ch = *text) != '\0' && + (match_slash || t_ch != '/')) { + if ((flags & WM_CASEFOLD) && ISUPPER(t_ch)) + t_ch = tolower(t_ch); + if (t_ch == p_ch) + break; + text++; + } + if (t_ch != p_ch) + return WM_NOMATCH; + } + if ((matched = dowild(p, text, flags)) != WM_NOMATCH) { + if (!match_slash || matched != WM_ABORT_TO_STARSTAR) + return matched; + } else if (!match_slash && t_ch == '/') + return WM_ABORT_TO_STARSTAR; + t_ch = *++text; + } + return WM_ABORT_ALL; + case '[': + p_ch = *++p; + if (p_ch == '^') + p_ch = '!'; + /* Assign literal 1/0 because of "matched" comparison. */ + negated = p_ch == '!' ? 1 : 0; + if (negated) { + /* Inverted character class. */ + p_ch = *++p; + } + prev_ch = 0; + matched = 0; + do { + if (!p_ch) + return WM_ABORT_ALL; + if (p_ch == '\\') { + p_ch = *++p; + if (!p_ch) + return WM_ABORT_ALL; + if (t_ch == p_ch) + matched = 1; + } else if (p_ch == '-' && prev_ch && p[1] && p[1] != ']') { + p_ch = *++p; + if (p_ch == '\\') { + p_ch = *++p; + if (!p_ch) + return WM_ABORT_ALL; + } + if (t_ch <= p_ch && t_ch >= prev_ch) + matched = 1; + else if ((flags & WM_CASEFOLD) && ISLOWER(t_ch)) { + uchar t_ch_upper = toupper(t_ch); + if (t_ch_upper <= p_ch && t_ch_upper >= prev_ch) + matched = 1; + } + p_ch = 0; /* This makes "prev_ch" get set to 0. */ + } else if (p_ch == '[' && p[1] == ':') { + const uchar *s; + int i; + for (s = p += 2; (p_ch = *p) && p_ch != ']'; p++) {} /*SHARED ITERATOR*/ + if (!p_ch) + return WM_ABORT_ALL; + i = p - s - 1; + if (i < 0 || p[-1] != ':') { + /* Didn't find ":]", so treat like a normal set. */ + p = s - 2; + p_ch = '['; + if (t_ch == p_ch) + matched = 1; + continue; + } + if (CC_EQ(s,i, "alnum")) { + if (ISALNUM(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "alpha")) { + if (ISALPHA(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "blank")) { + if (ISBLANK(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "cntrl")) { + if (ISCNTRL(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "digit")) { + if (ISDIGIT(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "graph")) { + if (ISGRAPH(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "lower")) { + if (ISLOWER(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "print")) { + if (ISPRINT(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "punct")) { + if (ISPUNCT(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "space")) { + if (ISSPACE(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "upper")) { + if (ISUPPER(t_ch)) + matched = 1; + else if ((flags & WM_CASEFOLD) && ISLOWER(t_ch)) + matched = 1; + } else if (CC_EQ(s,i, "xdigit")) { + if (ISXDIGIT(t_ch)) + matched = 1; + } else /* malformed [:class:] string */ + return WM_ABORT_ALL; + p_ch = 0; /* This makes "prev_ch" get set to 0. */ + } else if (t_ch == p_ch) + matched = 1; + } while (prev_ch = p_ch, (p_ch = *++p) != ']'); + if (matched == negated || + ((flags & WM_PATHNAME) && t_ch == '/')) + return WM_NOMATCH; + continue; + } + } + + return *text ? WM_NOMATCH : WM_MATCH; +} + +/* Match the "pattern" against the "text" string. */ +static int wildmatch(const char *pattern, const char *text, unsigned int flags) +{ + // local modification: move WM_CASEFOLD here + if (ignore_case) + flags |= WM_CASEFOLD; + + return dowild((const uchar*)pattern, (const uchar*)text, flags); +} + +// Copied from dir.h + +#define PATTERN_FLAG_NODIR 1 +#define PATTERN_FLAG_ENDSWITH 4 +#define PATTERN_FLAG_MUSTBEDIR 8 +#define PATTERN_FLAG_NEGATIVE 16 + +// Copied from dir.c + +static int fspathncmp(const char *a, const char *b, size_t count) +{ + return ignore_case ? strncasecmp(a, b, count) : strncmp(a, b, count); +} + +static int simple_length(const char *match) +{ + int len = -1; + + for (;;) { + unsigned char c = *match++; + len++; + if (c == '\0' || is_glob_special(c)) + return len; + } +} + +static int no_wildcard(const char *string) +{ + return string[simple_length(string)] == '\0'; +} + +static void parse_path_pattern(const char **pattern, + int *patternlen, + unsigned *flags, + int *nowildcardlen) +{ + const char *p = *pattern; + size_t i, len; + + *flags = 0; + if (*p == '!') { + *flags |= PATTERN_FLAG_NEGATIVE; + p++; + } + len = strlen(p); + if (len && p[len - 1] == '/') { + len--; + *flags |= PATTERN_FLAG_MUSTBEDIR; + } + for (i = 0; i < len; i++) { + if (p[i] == '/') + break; + } + if (i == len) + *flags |= PATTERN_FLAG_NODIR; + *nowildcardlen = simple_length(p); + /* + * we should have excluded the trailing slash from 'p' too, + * but that's one more allocation. Instead just make sure + * nowildcardlen does not exceed real patternlen + */ + if (*nowildcardlen > len) + *nowildcardlen = len; + if (*p == '*' && no_wildcard(p + 1)) + *flags |= PATTERN_FLAG_ENDSWITH; + *pattern = p; + *patternlen = len; +} + +static void trim_trailing_spaces(char *buf) +{ + char *p, *last_space = NULL; + + for (p = buf; *p; p++) + switch (*p) { + case ' ': + if (!last_space) + last_space = p; + break; + case '\\': + p++; + if (!*p) + return; + /* fallthrough */ + default: + last_space = NULL; + } + + if (last_space) + *last_space = '\0'; +} + +static int match_basename(const char *basename, int basenamelen, + const char *pattern, int prefix, int patternlen, + unsigned flags) +{ + if (prefix == patternlen) { + if (patternlen == basenamelen && + !fspathncmp(pattern, basename, basenamelen)) + return 1; + } else if (flags & PATTERN_FLAG_ENDSWITH) { + /* "*literal" matching against "fooliteral" */ + if (patternlen - 1 <= basenamelen && + !fspathncmp(pattern + 1, + basename + basenamelen - (patternlen - 1), + patternlen - 1)) + return 1; + } else { + // local modification: call wildmatch() directly + if (!wildmatch(pattern, basename, flags)) + return 1; + } + return 0; +} + +static int match_pathname(const char *pathname, int pathlen, + const char *base, int baselen, + const char *pattern, int prefix, int patternlen) +{ + // local modification: remove local variables + + /* + * match with FNM_PATHNAME; the pattern has base implicitly + * in front of it. + */ + if (*pattern == '/') { + pattern++; + patternlen--; + prefix--; + } + + /* + * baselen does not count the trailing slash. base[] may or + * may not end with a trailing slash though. + */ + if (pathlen < baselen + 1 || + (baselen && pathname[baselen] != '/') || + fspathncmp(pathname, base, baselen)) + return 0; + + // local modification: simplified because always baselen > 0 + pathname += baselen + 1; + pathlen -= baselen + 1; + + if (prefix) { + /* + * if the non-wildcard part is longer than the + * remaining pathname, surely it cannot match. + */ + if (prefix > pathlen) + return 0; + + if (fspathncmp(pattern, pathname, prefix)) + return 0; + pattern += prefix; + patternlen -= prefix; + pathname += prefix; + pathlen -= prefix; + + /* + * If the whole pattern did not have a wildcard, + * then our prefix match is all we need; we + * do not need to call fnmatch at all. + */ + if (!patternlen && !pathlen) + return 1; + } + + // local modification: call wildmatch() directly + return !wildmatch(pattern, pathname, WM_PATHNAME); +} + +// Copied from git/utf8.c + +static const char utf8_bom[] = "\357\273\277"; + +//----------------------------(IMPORT FROM GIT END)---------------------------- + +struct pattern { + unsigned int flags; + int nowildcardlen; + int patternlen; + int dirlen; + char pattern[]; +}; + +static struct pattern **pattern_list; +static int nr_patterns, alloced_patterns; + +// Remember the number of patterns at each directory level +static int *nr_patterns_at; +// Track the current/max directory level; +static int depth, max_depth; +static bool debug_on; +static FILE *out_fp, *stat_fp; +static char *prefix = ""; +static char *progname; + +static void __attribute__((noreturn)) perror_exit(const char *s) +{ + perror(s); + + exit(EXIT_FAILURE); +} + +static void __attribute__((noreturn)) error_exit(const char *fmt, ...) +{ + va_list args; + + fprintf(stderr, "%s: error: ", progname); + + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + + exit(EXIT_FAILURE); +} + +static void debug(const char *fmt, ...) +{ + va_list args; + int i; + + if (!debug_on) + return; + + fprintf(stderr, "[DEBUG] "); + + for (i = 0; i < depth * 2; i++) + fputc(' ', stderr); + + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); +} + +static void *xrealloc(void *ptr, size_t size) +{ + ptr = realloc(ptr, size); + if (!ptr) + perror_exit(progname); + + return ptr; +} + +static void *xmalloc(size_t size) +{ + return xrealloc(NULL, size); +} + +// similar to last_matching_pattern_from_list() in GIT +static bool is_ignored(const char *path, int pathlen, int dirlen, bool is_dir) +{ + int i; + + // Search in the reverse order because the last matching pattern wins. + for (i = nr_patterns - 1; i >= 0; i--) { + struct pattern *p = pattern_list[i]; + unsigned int flags = p->flags; + const char *gitignore_dir = p->pattern + p->patternlen + 1; + bool ignored; + + if ((flags & PATTERN_FLAG_MUSTBEDIR) && !is_dir) + continue; + + if (flags & PATTERN_FLAG_NODIR) { + if (!match_basename(path + dirlen + 1, + pathlen - dirlen - 1, + p->pattern, + p->nowildcardlen, + p->patternlen, + p->flags)) + continue; + } else { + if (!match_pathname(path, pathlen, + gitignore_dir, p->dirlen, + p->pattern, + p->nowildcardlen, + p->patternlen)) + continue; + } + + debug("%s: matches %s%s%s (%s/.gitignore)\n", path, + flags & PATTERN_FLAG_NEGATIVE ? "!" : "", p->pattern, + flags & PATTERN_FLAG_MUSTBEDIR ? "/" : "", + gitignore_dir); + + ignored = (flags & PATTERN_FLAG_NEGATIVE) == 0; + if (ignored) + debug("Ignore: %s\n", path); + + return ignored; + } + + debug("%s: no match\n", path); + + return false; +} + +static void add_pattern(const char *string, const char *dir, int dirlen) +{ + struct pattern *p; + int patternlen, nowildcardlen; + unsigned int flags; + + parse_path_pattern(&string, &patternlen, &flags, &nowildcardlen); + + if (patternlen == 0) + return; + + p = xmalloc(sizeof(*p) + patternlen + dirlen + 2); + + memcpy(p->pattern, string, patternlen); + p->pattern[patternlen] = 0; + memcpy(p->pattern + patternlen + 1, dir, dirlen); + p->pattern[patternlen + 1 + dirlen] = 0; + + p->patternlen = patternlen; + p->nowildcardlen = nowildcardlen; + p->dirlen = dirlen; + p->flags = flags; + + debug("Add pattern: %s%s%s\n", + flags & PATTERN_FLAG_NEGATIVE ? "!" : "", p->pattern, + flags & PATTERN_FLAG_MUSTBEDIR ? "/" : ""); + + if (nr_patterns >= alloced_patterns) { + alloced_patterns += 128; + pattern_list = xrealloc(pattern_list, + sizeof(*pattern_list) * alloced_patterns); + } + + pattern_list[nr_patterns++] = p; +} + +// similar to add_patterns_from_buffer() in GIT +static void add_patterns_from_gitignore(const char *dir, int dirlen) +{ + struct stat st; + char path[PATH_MAX], *buf, *entry; + size_t size; + int fd, pathlen, i; + + pathlen = snprintf(path, sizeof(path), "%s/.gitignore", dir); + if (pathlen >= sizeof(path)) + error_exit("%s: too long path was truncated\n", path); + + fd = open(path, O_RDONLY | O_NOFOLLOW); + if (fd < 0) { + if (errno != ENOENT) + return perror_exit(path); + return; + } + + if (fstat(fd, &st) < 0) + perror_exit(path); + + size = st.st_size; + + buf = xmalloc(size + 1); + if (read(fd, buf, st.st_size) != st.st_size) + perror_exit(path); + + buf[st.st_size] = '\n'; + if (close(fd)) + perror_exit(path); + + debug("Parse %s\n", path); + + entry = buf; + + // skip utf8 bom + if (!strncmp(entry, utf8_bom, strlen(utf8_bom))) + entry += strlen(utf8_bom); + + for (i = entry - buf; i < size; i++) { + if (buf[i] == '\n') { + if (entry != buf + i && entry[0] != '#') { + buf[i - (i && buf[i-1] == '\r')] = 0; + trim_trailing_spaces(entry); + add_pattern(entry, dir, dirlen); + } + entry = buf + i + 1; + } + } + + free(buf); +} + +// Save the current number of patterns and increment the depth +static void increment_depth(void) +{ + if (depth >= max_depth) { + max_depth += 1; + nr_patterns_at = xrealloc(nr_patterns_at, + sizeof(*nr_patterns_at) * max_depth); + } + + nr_patterns_at[depth] = nr_patterns; + depth++; +} + +// Decrement the depth, and free up the patterns of this directory level. +static void decrement_depth(void) +{ + depth--; + assert(depth >= 0); + + while (nr_patterns > nr_patterns_at[depth]) + free(pattern_list[--nr_patterns]); +} + +static void print_path(const char *path) +{ + // The path always starts with "./" + assert(strlen(path) >= 2); + + // Replace the root directory with a preferred prefix. + // This is useful for the tar command. + fprintf(out_fp, "%s%s\n", prefix, path + 2); +} + +static void print_stat(const char *path, struct stat *st) +{ + if (!stat_fp) + return; + + if (!S_ISREG(st->st_mode) && !S_ISLNK(st->st_mode)) + return; + + assert(strlen(path) >= 2); + + fprintf(stat_fp, "%c %9ld %10ld %s\n", + S_ISLNK(st->st_mode) ? 'l' : '-', + st->st_size, st->st_mtim.tv_sec, path + 2); +} + +// Traverse the entire directory tree, parsing .gitignore files. +// Print file paths that are not tracked by git. +// +// Return true if all files under the directory are ignored, false otherwise. +static bool traverse_directory(const char *dir, int dirlen) +{ + bool all_ignored = true; + DIR *dirp; + + debug("Enter[%d]: %s\n", depth, dir); + increment_depth(); + + add_patterns_from_gitignore(dir, dirlen); + + dirp = opendir(dir); + if (!dirp) + perror_exit(dir); + + while (1) { + struct dirent *d; + struct stat st; + char path[PATH_MAX]; + int pathlen; + bool ignored; + + errno = 0; + d = readdir(dirp); + if (!d) { + if (errno) + perror_exit(dir); + break; + } + + if (!strcmp(d->d_name, "..") || !strcmp(d->d_name, ".")) + continue; + + pathlen = snprintf(path, sizeof(path), "%s/%s", dir, d->d_name); + if (pathlen >= sizeof(path)) + error_exit("%s: too long path was truncated\n", path); + + if (lstat(path, &st) < 0) + perror_exit(path); + + if ((!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) || + is_ignored(path, pathlen, dirlen, S_ISDIR(st.st_mode))) { + ignored = true; + } else { + if (S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) + // If all the files in a directory are ignored, + // let's ignore that directory as well. This + // will avoid empty directories in the tarball. + ignored = traverse_directory(path, pathlen); + else + ignored = false; + } + + if (ignored) { + print_path(path); + } else { + print_stat(path, &st); + all_ignored = false; + } + } + + if (closedir(dirp)) + perror_exit(dir); + + decrement_depth(); + debug("Leave[%d]: %s\n", depth, dir); + + return all_ignored; +} + +static void usage(void) +{ + fprintf(stderr, + "usage: %s [options]\n" + "\n" + "Show files that are ignored by git\n" + "\n" + "options:\n" + " -d, --debug print debug messages to stderr\n" + " -e, --exclude PATTERN add the given exclude pattern\n" + " -h, --help show this help message and exit\n" + " -i, --ignore-case Ignore case differences between the patterns and the files\n" + " -o, --output FILE output the ignored files to a file (default: '-', i.e. stdout)\n" + " -p, --prefix PREFIX prefix added to each path (default: empty string)\n" + " -r, --rootdir DIR root of the source tree (default: current working directory)\n" + " -s, --stat FILE output the file stat of non-ignored files to a file\n", + progname); +} + +static void open_output(const char *pathname, FILE **fp) +{ + if (strcmp(pathname, "-")) { + *fp = fopen(pathname, "w"); + if (!*fp) + perror_exit(pathname); + } else { + *fp = stdout; + } +} + +static void close_output(const char *pathname, FILE *fp) +{ + fflush(fp); + + if (ferror(fp)) + error_exit("not all data was written to the output\n"); + + if (fclose(fp)) + perror_exit(pathname); +} + +int main(int argc, char *argv[]) +{ + const char *output = "-"; + const char *rootdir = "."; + const char *stat = NULL; + + progname = strrchr(argv[0], '/'); + if (progname) + progname++; + else + progname = argv[0]; + + while (1) { + static struct option long_options[] = { + {"debug", no_argument, NULL, 'd'}, + {"help", no_argument, NULL, 'h'}, + {"ignore-case", no_argument, NULL, 'i'}, + {"output", required_argument, NULL, 'o'}, + {"prefix", required_argument, NULL, 'p'}, + {"rootdir", required_argument, NULL, 'r'}, + {"stat", required_argument, NULL, 's'}, + {"exclude", required_argument, NULL, 'x'}, + {}, + }; + + int c = getopt_long(argc, argv, "dhino:p:r:s:x:", long_options, NULL); + + if (c == -1) + break; + + switch (c) { + case 'd': + debug_on = true; + break; + case 'h': + usage(); + exit(0); + case 'i': + ignore_case = true; + break; + case 'o': + output = optarg; + break; + case 'p': + prefix = optarg; + break; + case 'r': + rootdir = optarg; + break; + case 's': + stat = optarg; + break; + case 'x': + add_pattern(optarg, ".", strlen(".")); + break; + case '?': + usage(); + /* fallthrough */ + default: + exit(EXIT_FAILURE); + } + } + + open_output(output, &out_fp); + if (stat && stat[0]) + open_output(stat, &stat_fp); + + if (chdir(rootdir)) + perror_exit(rootdir); + + add_pattern(".git/", ".", strlen(".")); + + if (traverse_directory(".", strlen("."))) + print_path("./"); + + assert(depth == 0); + + while (nr_patterns > 0) + free(pattern_list[--nr_patterns]); + free(pattern_list); + free(nr_patterns_at); + + close_output(output, out_fp); + if (stat_fp) + close_output(stat, stat_fp); + + return 0; +} -- cgit 1.2.3-korg