Dear Maintainer, I'm working around #35733 and #468333 by renaming *.rlib to *.rlib.a before running dh_strip(1). This works well, however one file is omitted by debhelper: ---- $ xxd debian/libstd-rust-dev/usr/lib/rustlib/x86_64-unknown-linux-gnu/lib/libstd-b6402de156fe6ac9.rlib | head -n16 00000000: 213c 6172 6368 3e0a 2f20 2020 2020 2020 !<arch>./ 00000010: 2020 2020 2020 2020 3020 2020 2020 2020 0 00000020: 2020 2020 3020 2020 2020 3020 2020 2020 0 0 00000030: 3020 2020 2020 2020 3132 3037 3634 2020 0 120764 00000040: 2020 600a 0000 0511 0001 d892 0001 d892 `............. 00000050: 0001 d892 0001 d892 0001 d892 0001 d892 ................ 00000060: 0001 d892 0001 d892 0001 d892 0001 d892 ................ 00000070: 0001 d892 0001 d892 0001 d892 0001 d892 ................ 00000080: 0001 d892 0001 d892 0001 d892 0001 d892 ................ 00000090: 0001 d892 0001 d892 0001 d892 0001 d892 ................ 000000a0: 0001 d892 0001 d892 0001 d892 0001 d892 ................ 000000b0: 0001 d892 0001 d892 0001 d892 0001 d892 ................ 000000c0: 0001 d892 0001 d892 0001 d892 0001 d892 ................ 000000d0: 0001 d892 0001 d892 0001 d892 0001 d892 ................ 000000e0: 0001 d892 0001 d892 0001 d892 0001 d892 ................ 000000f0: 0001 d892 0001 d892 0001 d892 0001 d892 ................ ---- This is because dh_strip uses perl's -B check to filter out "text" files for stripping, see here:---- # Is it a static library, and not a debug library? if ($fn =~ m/\/lib[^\/]*\.a$/ && $fn !~ m/.*_g\.a$/) { # Is it a binary file, or something else (maybe a linker # script on Hurd, for example? I don't use file, because # file returns a variety of things on static libraries. if (-B $fn) { push @static_libs, $fn; return; } } ---- This fails for the libstd-*.rlib I just gave - but if I remove the -B check everything works. Obviously I can't do that in the packaging files however. Is there another way I can work around this issue, before it is fixed? X
Ximin Luo: $ perl -e 'print((-B "libstd.head") ? "yes\n" : "no\n");' no $ perl -e 'print((-B "libcore.head") ? "yes\n" : "no\n");' yes More info here: https://eli.thegreenplace.net/2011/10/19/perls-guess-if-file-is-text-or-binary-implemented-in-python X
I face the same problem - a static library built on Jan 28th was stripped correctly, but one built on March 26th is now ignored by dh_strip because perl thinks it's a text file. The library has no diff between the two builds. Once manually stripped, diffoscope reports this difference between the two libraries:--- librte_pmd_virtio_crypto.a.old +++ librte_pmd_virtio_crypto.a ├── nm -s {} │ @@ -114,44 +114,44 @@ │ 000000000000012c r .LC82 │ 0000000000000143 r .LC83 │ 000000000000015d r .LC84 │ 0000000000000172 r .LC85 │ 0000000000000187 r .LC86 │ 00000000000000c8 r .LC9 │ U _GLOBAL_OFFSET_TABLE_ │ -00000000000002d0 r __func__.37667 │ -00000000000002f0 r __func__.39290 │ -0000000000000170 r __func__.39787 │ -00000000000004a0 r __func__.39809 │ -0000000000000480 r __func__.39827 │ -0000000000000040 r __func__.39845 │ -0000000000000400 r __func__.39851 │ -00000000000000a0 r __func__.39865 │ -00000000000000c0 r __func__.39875 │ -00000000000000e0 r __func__.39881 │ -0000000000000100 r __func__.39895 │ -0000000000000120 r __func__.39901 │ -0000000000000000 r __func__.39907 │ -0000000000000020 r __func__.39915 │ -00000000000003c0 r __func__.39923 │ -0000000000000460 r __func__.39928 │ -0000000000000060 r __func__.39934 │ -0000000000000440 r __func__.39939 │ -0000000000000420 r __func__.39949 │ -0000000000000140 r __func__.39960 │ -0000000000000320 r __func__.39965 │ -0000000000000360 r __func__.39970 │ -00000000000003a0 r __func__.39988 │ -00000000000001e0 r __func__.40023 │ -00000000000001a0 r __func__.40038 │ -0000000000000220 r __func__.40053 │ -0000000000000260 r __func__.40061 │ -00000000000002a0 r __func__.40078 │ -0000000000000080 r __func__.40090 │ -00000000000003e0 r __func__.40097 │ +00000000000002d0 r __func__.37686 │ +00000000000002f0 r __func__.39309 │ +0000000000000170 r __func__.39806 │ +00000000000004a0 r __func__.39828 │ +0000000000000480 r __func__.39846 │ +0000000000000040 r __func__.39864 │ +0000000000000400 r __func__.39870 │ +00000000000000a0 r __func__.39884 │ +00000000000000c0 r __func__.39894 │ +00000000000000e0 r __func__.39900 │ +0000000000000100 r __func__.39914 │ +0000000000000120 r __func__.39920 │ +0000000000000000 r __func__.39926 │ +0000000000000020 r __func__.39934 │ +00000000000003c0 r __func__.39942 │ +0000000000000460 r __func__.39947 │ +0000000000000060 r __func__.39953 │ +0000000000000440 r __func__.39958 │ +0000000000000420 r __func__.39968 │ +0000000000000140 r __func__.39979 │ +0000000000000320 r __func__.39984 │ +0000000000000360 r __func__.39989 │ +00000000000003a0 r __func__.40007 │ +00000000000001e0 r __func__.40042 │ +00000000000001a0 r __func__.40057 │ +0000000000000220 r __func__.40072 │ +0000000000000260 r __func__.40080 │ +00000000000002a0 r __func__.40097 │ +0000000000000080 r __func__.40109 │ +00000000000003e0 r __func__.40116 │ U __rte_panic │ U __snprintf_chk │ U __stack_chk_fail │ U __tls_get_addr │ U crypto_virtio_hw_internal │ 0000000000000fea t crypto_virtio_pci_probe │ 0000000000000050 t crypto_virtio_pci_remove │ @@ -251,19 +251,19 @@ │ 000000000000003c r .LC3 │ 0000000000000028 r .LC4 │ 0000000000000060 r .LC5 │ 000000000000005a r .LC7 │ 0000000000000098 r .LC8 │ 00000000000000c8 r .LC9 │ U _GLOBAL_OFFSET_TABLE_ │ -0000000000000050 r __func__.38928 │ -0000000000000070 r __func__.39003 │ -0000000000000000 r __func__.39054 │ -0000000000000010 r __func__.39062 │ -0000000000000030 r __func__.39077 │ +0000000000000050 r __func__.38949 │ +0000000000000070 r __func__.39024 │ +0000000000000000 r __func__.39075 │ +0000000000000010 r __func__.39083 │ +0000000000000030 r __func__.39098 │ U __stack_chk_fail │ 0000000000000400 C crypto_virtio_hw_internal │ 000000000000002b t get_cfg_addr │ 0000000000000190 t modern_del_queue │ 0000000000000080 t modern_get_features │ 0000000000000120 t modern_get_isr │ 0000000000000170 t modern_get_queue_num │ @@ -307,23 +307,23 @@ │ 0000000000000010 r .LC2 │ 0000000000000020 r .LC3 │ 0000000000000060 r .LC4 │ 0000000000000080 r .LC5 │ 0000000000000110 r .LC7 │ 0000000000000016 r .LC8 │ U _GLOBAL_OFFSET_TABLE_ │ -00000000000000d0 r __func__.36564 │ -00000000000000f0 r __func__.38828 │ -0000000000000060 r __func__.38875 │ -0000000000000090 r __func__.38887 │ -0000000000000150 r __func__.38896 │ -0000000000000170 r __func__.38906 │ -0000000000000130 r __func__.38912 │ -0000000000000110 r __func__.38927 │ -00000000000000b0 r __func__.38936 │ +00000000000000d0 r __func__.36581 │ +00000000000000f0 r __func__.38845 │ +0000000000000060 r __func__.38892 │ +0000000000000090 r __func__.38904 │ +0000000000000150 r __func__.38913 │ +0000000000000170 r __func__.38923 │ +0000000000000130 r __func__.38929 │ +0000000000000110 r __func__.38944 │ +00000000000000b0 r __func__.38953 │ U __rte_panic │ U __stack_chk_fail │ U __tls_get_addr │ U crypto_virtio_hw_internal │ U cryptodev_virtio_driver_id │ U per_lcore__lcore_id │ U rte_log │ @@ -343,15 +343,15 @@ │ U virtqueue_disable_intr │ │ crypto_virtio_virtqueue.c.o: │ 0000000000000000 r .LC0 │ 0000000000000000 r .LC1 │ 0000000000000040 r .LC2 │ U _GLOBAL_OFFSET_TABLE_ │ -0000000000000000 r __func__.36474 │ +0000000000000000 r __func__.36491 │ U __rte_panic │ U __stack_chk_fail │ U __tls_get_addr │ U per_lcore__lcore_id │ 0000000000000000 t rte_mempool_ops_enqueue_bulk.part.0 │ U rte_mempool_ops_table │ 0000000000000000 t rte_pktmbuf_free ├── filetype from diffoscope │ @@ -1 +1 @@ │ -ArFile │ +StaticLibFile Not sure how it detects the file type, and why it changes from ArFile to StaticLibFile (or what's the difference between the two). Old package with librte_pmd_virtio_crypto.a stripped correctly: http://snapshot.debian.org/package/dpdk/19.11-3/#libdpdk-dev_19.11-3 New package with librte_pmd_virtio_crypto.a unstripped: https://salsa.debian.org/paelzer-guest/dpdk/-/jobs/630625/artifacts/file/debian/output/libdpdk-dev_19.11.1-1+salsaci_amd64.deb
Control: tags -1 patch Opened MR on Salsa to add a fallback on file --mime-type when perl -B fails: https://salsa.debian.org/debian/debhelper/-/merge_requests/37
Luca Boccassi: Hi Luca, Thanks for the proposed patch. At this point, I would rather slowly move away from the use of file(1) and am more interested in a patch that avoided new uses of file. The primary issue is that debhelper's design is not a good match with programs with a "slow" upstart time. This leads to "weird" work arounds to avoid the "penality" for running the program (e.g. as was done with ELF binaries until compat 12 where we finally started doing the right thing). ~Niels
Hi, I understand, and fully agree with the goal. However I'm really not sure how to go and fix Perl's broken heuristics - it sounds really scary :-) And I can't think of any other way to fix it at the moment. Should I avoid dh_strip in the affected package? Or is there any other way? Thanks! Kind regards, Luca Boccassi
Chiming in here, as I am unfortunately affected by this relatively rare
problem.
Would it be possible to use libfile-libmagic-perl here and in other
cases where debhelper currently uses file(1)? At least that would save
the overhead of process creation.
Cheers,
Sven
Hi! I don't know why, but for some reason this started affecting MariaDB recently. Perhaps something related to the new MariaDB 11.8.1 itself, or a some build tool updated in Debian unstable recently. Details and workaround described in https://salsa.debian.org/otto/mariadb-server/-/commit/398a34c520351d81fd2d095611d156f34ff95a06
This bug bit me in packaging libcec. Lintian says unstripped-static-library, which dh_strip should be stripping but doesn't. I added a stanza to libcec debian/rules to strip the file explicitly instead of relying on dh_strip, but that's annoying. Had claude look at the issue. It independently found the present bug in dh_strip. And then it found this particular bug report. But it also (a) did a deeper analysis of the bug, and (b) suggested a simpler fix, namely having the perl code for dh_strip directly check the file magic number before switching over to heuristics. For your AI condescension pleasure I'm including its long claude-speak discourse on the matter below. This includes a deeper analysis of the bug and a tiny patch. Seriously: despite claude acting like a know-it-all we'd all like to strangle, it seemed to nail the diagnosis and a proper fix. And this dh_strip bug is stupid and annoying and causing technical debt elsewhere in the project so really should be fixed. Cheers,--------------------------------------- -- BEGIN CLAUDE SPEAK -- --------------------------------------- [This is a follow-up to the existing #875780 rather than a new report: that bug has had the symptom since 2017 but, as far as I can see from the log, never the mechanism. Below is the mechanism, a one-line test for it, and a patch that needs neither file(1) nor libmagic, which I think answers the objections that stalled the earlier attempts.] Package: debhelper Version: 14.3 Severity: normal I hit this in src:libcec, where libcec-dev shipped a 22MB libcec.a with full debug info. lintian says unstripped-static-library; dh_strip issues no strip command for the file at all. The cause is the `-B` guard in dh_strip's testfile(): # Is it a static library, and not a debug library? if ($fn =~ m/\/lib[^\/]*\.a$/ && $fn !~ m/.*_g\.a$/) { # Is it a binary file, or something else (maybe a linker # script on Hurd, for example? I don't use file, because # file returns a variety of things on static libraries. if (-B $fn) { push @static_libs, $fn; return; } } The filename regex matches. `-B` is what fails. WHY -B FAILS Earlier messages in this bug describe perl's -T/-B as a statistical heuristic over "odd" characters, and reason about whether an archive has enough of them. That is only the second half of the rule. perlfunc: The -T and -B tests work as follows. The first block or so of the file is examined to see if it is valid UTF-8 that includes non-ASCII characters. If so, it's a -T file. Otherwise, that same portion of the file is examined for odd characters such as strange control codes or characters with the high bit set. If more than a third of the characters are strange, it's a -B file; otherwise it's a -T file. So the odd-character count is only reached if the UTF-8 test fails. It is the UTF-8 branch that fires here, and it fires on data that is overwhelmingly "odd": $ perl -e 'printf "size=%d -B=%d -T=%d\n", -s $ARGV[0], (-B $ARGV[0])?1:0, (-T $ARGV[0])?1:0' libcec.a size=22656606 -B=0 -T=1 443 of the first 512 bytes are high-bit or control characters, i.e. the fallback rule would have said "binary" with room to spare. The reason the block is valid UTF-8 is the ar symbol table. Its entries are 4-byte big-endian member offsets, and libcec.a's first member sits at 0x0001CD8E, so the table is this, over and over: 0000100 ` \n \0 \0 \b \0 \0 001 315 216 \0 001 315 216 0000120 \0 001 315 216 \0 001 315 216 \0 001 315 216 \0 001 315 216 * 0xCD 0x8E is a well-formed two-byte sequence (U+034E), 0x00 and 0x01 are ASCII, and the ar header preceding it is all ASCII. The whole 512-byte block therefore decodes: $ perl -MEncode -e 'open(F,"<",$ARGV[0]); binmode F; read(F,$b,512); print eval { Encode::decode("UTF-8",$b,Encode::FB_CROAK); 1 } ? "valid UTF-8\n" : "not UTF-8\n"' libcec.a valid UTF-8 To confirm that this branch, and not the odd-character count, is what decides the outcome, take the same block and corrupt one continuation byte into an invalid lead byte. The high-bit count is unchanged; only UTF-8 validity changes: A (block as-is) odd=443/512 -B=0 -T=1 B (one byte -> 0xC0) odd=443/512 -B=1 -T=0 (perlfunc separately says a file with a zero byte in the examined portion is considered binary. That is not what the implementation does: the block above is full of NULs and still comes out -T. The UTF-8 test wins. Worth knowing before assuming a NUL-check would be enough here.) So this is not "archives are marginally texty and sometimes tip over the 1/3 threshold". It is a hard switch: an archive is misclassified exactly when its symbol table's member offsets happen to land in byte ranges that form valid UTF-8. That is a function of member sizes, which is why Ximin Luo's original report saw libcore.rlib and libstd.rlib disagree, why MariaDB's libraries "changed from ArFile to StaticLibFile" between build dates, and why the bug looks nondeterministic. It is perfectly deterministic; it just depends on data no one controls deliberately. PROPOSED FIX ar archives have an 8-byte magic. Reading it is cheaper than -B (which reads a whole block and scans it), needs no subprocess, and adds no dependency -- so it sidesteps both Niels' objection to file(1) and the libfile-libmagic-perl suggestion. The helper below is deliberately written in the same shape as the existing is_so_or_exec_elf_file(), and gives the guard the semantics its comment always claimed. It also keeps the case that guard exists for: a Hurd linker script named libfoo.a has no ar magic and is still skipped.--- a/lib/Debian/Debhelper/Dh_Lib.pm +++ b/lib/Debian/Debhelper/Dh_Lib.pm @@ -151,6 +151,7 @@ rm_files excludefile is_so_or_exec_elf_file + is_ar_archive is_empty_dir reset_perm_and_owner log_installed_files @@ -3161,6 +3162,23 @@ ELF_TYPE_SHARED_OBJECT => 0x0003, }; +use constant AR_MAGIC => "!<arch>\n"; + +sub is_ar_archive { + my ($file) = @_; + open(my $fd, '<:raw', $file) or error("open $file: $!"); + my $buflen = 0; + my $buf; + while ($buflen < length(AR_MAGIC)) { + my $r = read($fd, $buf, length(AR_MAGIC) - $buflen, $buflen) // error("read ($file): $!"); + last if $r == 0; # EOF + $buflen += $r; + } + close($fd); + return 0 if $buflen < length(AR_MAGIC); + return $buf eq AR_MAGIC ? 1 : 0; +} + sub is_so_or_exec_elf_file { my ($file) = @_; open(my $fd, '<:raw', $file) or error("open $file: $!"); --- a/dh_strip +++ b/dh_strip @@ -239,10 +239,9 @@ } # Is it a static library, and not a debug library? if ($fn =~ m/\/lib[^\/]*\.a$/ && $fn !~ m/.*_g\.a$/) { - # Is it a binary file, or something else (maybe a linker - # script on Hurd, for example? I don't use file, because - # file returns a variety of things on static libraries. - if (-B $fn) { + # Is it really an ar archive, or something else (maybe a + # linker script on Hurd, for example)? + if (is_ar_archive($fn)) { push @static_libs, $fn; return; } TESTING Package tree containing the real libcec.a plus a fake Hurd-style linker script named libscript.a. Same tree, both runs, debhelper 14.3, perl 5.42.2. stock dh_strip -v: (no strip command issued) libcec.a 22656606 -> 22656606 libscript.a 65 -> 65 patched dh_strip -v: strip --strip-debug --remove-section=.comment --remove-section=.note --enable-deterministic-archives -R .gnu.lto_* -R .gnu.debuglto_* -N __gnu_lto_slim -N __gnu_lto_v1 .../libcec.a libcec.a 22656606 -> 2315086 libscript.a 65 -> 65 So the archive gets stripped, the linker script is still left alone, and lintian's unstripped-static-library goes away. In the real package this took libcec-dev from 2.9MB to 330kB. NOTE ON SCOPE The same misclassification will apply to anything else gated on -B. I have only verified the static library path. Rust .rlib files are also ar archives and were mentioned upstream in this bug, but they do not match dh_strip's /lib[^\/]*\.a$/ pattern, so whatever affects them is a separate question from this one.