Hi,

I happened to look at the llvm-toolchain-18/19 reproducibility results
because they seem to be flaky. I noticed that the diffoscope output [1]
showed differences in assembly code that to my untrained eyes suggest
that the binaries of llvm-toolchain-18/19 might depend on the CPU
capabilities of the build cpu, which might be a baseline violation. Can
you please check?

It shows calls like pand vs vpand and movd vs vmovd. A quick query on
the internets seems to indicate that compiling with -mno-avx could
prevent this delta.

Paul

https://tests.reproducible-builds.org/debian/rb-pkg/unstable/amd64/diffoscope-results/llvm-toolchain-19.html

objdump --line-numbers --disassemble --demangle --reloc
--no-show-raw-insn --section=.text.acosf {}
Offset 7, 200 lines modified	Offset 7, 225 lines modified
7 	acosf():	7 	acosf():
8 	»       endbr64	8 	»       endbr64
9 	»       push···%rbp	9 	»       push···%rbp
10 	»       mov····%rsp,%rbp	10 	»       mov····%rsp,%rbp
11 	»       sub····$0x30,%rsp	11 	»       sub····$0x30,%rsp
12 	»       mov····%fs:0x28,%rax	12 	»       mov····%fs:0x28,%rax
13 	»       mov····%rax,-0x8(%rbp)	13 	»       mov····%rax,-0x8(%rbp)
  	14 	»       movd···%xmm0,%eax
14 	»       vpbroadcastd·0x0(%rip),%xmm1········	15 	»
movdqa·0x0(%rip),%xmm1········
15 	·R_X86_64_PC32»  .LCPI0_0-0x4	16 	·R_X86_64_PC32»  .LCPI0_0-0x4
16 	»       vmovd··%xmm0,%eax
17 	»       vpand··%xmm1,%xmm0,%xmm1	17 	»       pand···%xmm0,%xmm1
18 	»       vmovd··%xmm1,%ecx	18 	»       movd···%xmm1,%ecx
19 	»       cmp····$0x3f000000,%ecx	19 	»       cmp····$0x3f000000,%ecx
20 	»       ja·····6f·<__llvm_libc_19_1_1_::acosf(float)+0x6f>	20 	»
   ja·····6a·<__llvm_libc_19_1_1_::acosf(float)+0x6a>
21 	»       cmp····$0x3a7fffff,%ecx	21 	»       cmp····$0x3a7fffff,%ecx
22 	»       ja·····c3·<__llvm_libc_19_1_1_::acosf(float)+0xc3>	22 	»
   ja·····ab·<__llvm_libc_19_1_1_::acosf(float)+0xab>
23 	»       cmp····$0x328885a2,%eax	23 	»       cmp····$0x328885a2,%eax
24 	»       jg·····1e6·<__llvm_libc_19_1_1_::acosf(float)+0x1e6>	24 	»
     jg·····225·<__llvm_libc_19_1_1_::acosf(float)+0x225>
25 	»       cmp····$0xb28885a3,%eax	25 	»       cmp····$0xb28885a3,%eax
26 	»       je·····2c9·<__llvm_libc_19_1_1_::acosf(float)+0x2c9>	26 	»
     je·····2ed·<__llvm_libc_19_1_1_::acosf(float)+0x2ed>
27 	»       cmp····$0xb9826222,%eax	27 	»       cmp····$0xb9826222,%eax
28 	»       jne····28f·<__llvm_libc_19_1_1_::acosf(float)+0x28f>	28 	»
     jne····2bc·<__llvm_libc_19_1_1_::acosf(float)+0x2bc>
29 	»       lea····0x0(%rip),%rax········	29 	»
lea····0x0(%rip),%rax········
30 	·R_X86_64_PC32»
.rodata._ZN19__llvm_libc_19_1_1_L13ACOSF_EXCEPTSE+0x38	30
·R_X86_64_PC32»  .rodata._ZN19__llvm_libc_19_1_1_L13ACOSF_EXCEPTSE+0x38
31 	»       jmp····2d9·<__llvm_libc_19_1_1_::acosf(float)+0x2d9>	31 	»
     jmp····2fd·<__llvm_libc_19_1_1_::acosf(float)+0x2fd>
32 	»       cmp····$0x3f800000,%ecx	32 	»       cmp····$0x3f800000,%ecx
33 	»       jb·····146·<__llvm_libc_19_1_1_::acosf(float)+0x146>	33 	»
     jb·····15d·<__llvm_libc_19_1_1_::acosf(float)+0x15d>
34 	»       jne····208·<__llvm_libc_19_1_1_::acosf(float)+0x208>	34 	»
     jne····247·<__llvm_libc_19_1_1_::acosf(float)+0x247>
35 	»       vxorps·%xmm0,%xmm0,%xmm0	35 	»       pxor···%xmm0,%xmm0
36 	»       test···%eax,%eax	36 	»       test···%eax,%eax
37 	»       jns····2b4·<__llvm_libc_19_1_1_::acosf(float)+0x2b4>	37 	»
     jns····347·<__llvm_libc_19_1_1_::acosf(float)+0x347>
38 	»       movl···$0x40490fdb,-0x28(%rbp)	38 	»
movl···$0x40490fdb,-0x28(%rbp)
39 	»       vmovss·-0x28(%rbp),%xmm0	39 	»       movss··-0x28(%rbp),%xmm0
40 	»       vaddss·0x0(%rip),%xmm0,%xmm0········	40 	»
addss··0x0(%rip),%xmm0········

clang doesn't have a good way to set the baseline for a build, nothing
as easy as in GCC with --with-arch=<baseline>.

I did that for both the armel and armhf architectures (see the
arm32-defaults.diff patch), but this is unchecked for any other release
architectures, and obviously wrong for some architectures.

Please could you run (at least on amd64)

/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 --help

and look for the last lines like

Subdirectories of glibc-hwcaps directories, in priority order:
   x86-64-v4 (supported, searched)
   x86-64-v3 (supported, searched)
   x86-64-v2 (supported, searched)

on the machines where you saw that?

Just making sure everybody, esp. both bugs, got the message.

just a note that if you need me to run something on those opterons, then
please hurry, I converted all our opterons except one to xeon yesterday
and now only have access to one opteron machine left, which I might loose
next week and will quite definitly loose within the next 30 days.