[BOLT] Synchronize function and section order (#172419)
The order in which functions are emitted into the output file does not
always reflect the order in which they will appear in the final file.
This discrepancy occurs because code is emitted to different sections,
such as `.text`, `.text.cold`, `.text.mover`, etc.
To make passes that rely on the relative function order - such as
`LongJmpPass` - more precise and functional, sort the output functions
to reflect their final layout, and validate the layout after code
sections are sorted.
Note that, at this time, we only directly change the order of the main
fragments of functions. The order of other fragments, such as cold and
warm sections relative to other fragment types, is determined by the
section order.
diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp
index 35e30a3..480d0ce 100644
--- a/bolt/lib/Passes/BinaryPasses.cpp
+++ b/bolt/lib/Passes/BinaryPasses.cpp
@@ -573,6 +573,19 @@
llvm::copy(BC.getInjectedBinaryFunctions(),
std::back_inserter(OutputFunctions));
+ // Place hot text movers in front.
+ if (opts::HotText) {
+ std::stable_partition(
+ OutputFunctions.begin(), OutputFunctions.end(),
+ [](const BinaryFunction *A) { return opts::isHotTextMover(*A); });
+ }
+
+ if (opts::HotFunctionsAtEnd) {
+ std::stable_partition(
+ OutputFunctions.begin(), OutputFunctions.end(),
+ [](const BinaryFunction *A) { return !A->hasValidIndex(); });
+ }
+
return Error::success();
}
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index eed2c20..af21c9d 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -4079,11 +4079,13 @@
: (A->getName() < B->getName());
}
- // Place movers before anything else.
- if (A->getName() == BC->getHotTextMoverSectionName())
- return true;
- if (B->getName() == BC->getHotTextMoverSectionName())
- return false;
+ // Place hot text movers before anything else.
+ if (opts::HotText) {
+ if (A->getName() == BC->getHotTextMoverSectionName())
+ return true;
+ if (B->getName() == BC->getHotTextMoverSectionName())
+ return false;
+ }
// Depending on opts::HotFunctionsAtEnd, place main and warm sections in
// order.
@@ -4105,6 +4107,28 @@
// Determine the order of sections.
llvm::stable_sort(CodeSections, compareSections);
+#ifndef NDEBUG
+ // Verify that the order of sections and functions is consistent.
+ uint32_t Index = 1;
+ for (BinarySection *Sec : CodeSections)
+ Sec->setIndex(Index++);
+
+ uint32_t LastIndex = 0;
+ for (const BinaryFunction *BF : BC->getOutputBinaryFunctions()) {
+ if (!BF->isEmitted() || BF->isPatch())
+ continue;
+
+ ErrorOr<BinarySection &> Sec = BF->getCodeSection();
+ if (!Sec)
+ continue;
+
+ assert(Sec->getIndex() >= LastIndex &&
+ "Section order does not match function order");
+
+ LastIndex = Sec->getIndex();
+ }
+#endif
+
return CodeSections;
}
diff --git a/bolt/test/code-at-high-address.c b/bolt/test/code-at-high-address.c
index fa33c1e..0cdfe4f 100644
--- a/bolt/test/code-at-high-address.c
+++ b/bolt/test/code-at-high-address.c
@@ -5,9 +5,10 @@
// RUN: %clang %cflags -O0 %s -o %t -no-pie -Wl,-q -falign-functions=64 \
// RUN: -nostartfiles -nostdlib -ffreestanding
-// RUN: llvm-bolt %t -o %t.bolt --use-old-text --align-functions=1 \
-// RUN: --no-huge-pages --align-text=1 --use-gnu-stack --hot-functions-at-end \
-// RUN: | FileCheck %s --check-prefix=CHECK-BOLT
+// RUN: link_fdata %s %t %t.fdata
+// RUN: llvm-bolt %t -o %t.bolt --data %t.fdata --use-old-text \
+// RUN: --align-functions=1 --no-huge-pages --align-text=1 --use-gnu-stack \
+// RUN: --hot-functions-at-end | FileCheck %s --check-prefix=CHECK-BOLT
// RUN: llvm-readelf --sections %t.bolt | FileCheck %s
// CHECK-BOLT: using original .text for new code with 0x1 alignment at {{.*}}
@@ -17,8 +18,14 @@
// CHECK: .bolt.org.text PROGBITS
// CHECK-NOT: {{ 000000 }}
// CHECK-SAME: AX
+// CHECK-NEXT: .text.cold PROGBITS
// CHECK-NEXT: .text PROGBITS
+// FDATA: 0 [unknown] 0 1 foo 0 0 1
int foo() { return 0; }
-int main() { return foo(); }
+// Cold function.
+int bar() { return 42; }
+
+// FDATA: 0 [unknown] 0 1 main 0 0 1
+int main(int argc, char **argv) { return argc ? foo() : bar(); }
