blob: 93faabad5e0b5862bc74677ea2ec8e410fc2ae6a [file] [log] [blame]
commit 2240d72f15f3b7b9d9fb65450f9bf635fd310f6f
Author: Nick Desaulniers <ndesaulniers@google.com>
Date: Tue Jul 12 09:17:15 2022 -0700
[X86] initial -mfunction-return=thunk-extern support
Adds support for:
* `-mfunction-return=<value>` command line flag, and
* `__attribute__((function_return("<value>")))` function attribute
Where the supported <value>s are:
* keep (disable)
* thunk-extern (enable)
thunk-extern enables clang to change ret instructions into jmps to an
external symbol named __x86_return_thunk, implemented as a new
MachineFunctionPass named "x86-return-thunks", keyed off the new IR
attribute fn_ret_thunk_extern.
The symbol __x86_return_thunk is expected to be provided by the runtime
the compiled code is linked against and is not defined by the compiler.
Enabling this option alone doesn't provide mitigations without
corresponding definitions of __x86_return_thunk!
This new MachineFunctionPass is very similar to "x86-lvi-ret".
The <value>s "thunk" and "thunk-inline" are currently unsupported. It's
not clear yet that they are necessary: whether the thunk pattern they
would emit is beneficial or used anywhere.
Should the <value>s "thunk" and "thunk-inline" become necessary,
x86-return-thunks could probably be merged into x86-retpoline-thunks
which has pre-existing machinery for emitting thunks (which could be
used to implement the <value> "thunk").
Has been found to build+boot with corresponding Linux
kernel patches. This helps the Linux kernel mitigate RETBLEED.
* CVE-2022-23816
* CVE-2022-28693
* CVE-2022-29901
See also:
* "RETBLEED: Arbitrary Speculative Code Execution with Return
Instructions."
* AMD SECURITY NOTICE AMD-SN-1037: AMD CPU Branch Type Confusion
* TECHNICAL GUIDANCE FOR MITIGATING BRANCH TYPE CONFUSION REVISION 1.0
2022-07-12
* Return Stack Buffer Underflow / Return Stack Buffer Underflow /
CVE-2022-29901, CVE-2022-28693 / INTEL-SA-00702
SystemZ may eventually want to support "thunk-extern" and "thunk"; both
options are used by the Linux kernel's CONFIG_EXPOLINE.
This functionality has been available in GCC since the 8.1 release, and
was backported to the 7.3 release.
Many thanks for folks that provided discrete review off list due to the
embargoed nature of this hardware vulnerability. Many Bothans died to
bring us this information.
Link: https://www.youtube.com/watch?v=IF6HbCKQHK8
Link: https://github.com/llvm/llvm-project/issues/54404
Link: https://gcc.gnu.org/legacy-ml/gcc-patches/2018-01/msg01197.html
Link: https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/advisory-guidance/return-stack-buffer-underflow.html
Link: https://arstechnica.com/information-technology/2022/07/intel-and-amd-cpus-vulnerable-to-a-new-speculative-execution-attack/?comments=1
Link: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=ce114c866860aa9eae3f50974efc68241186ba60
Link: https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00702.html
Link: https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00707.html
Reviewed By: aaron.ballman, craig.topper
Differential Revision: https://reviews.llvm.org/D129572
diff --git a/clang/docs/ReleaseNotes.rst.rej b/clang/docs/ReleaseNotes.rst.rej
new file mode 100644
index 000000000000..b6e831298d40
--- /dev/null
+++ b/clang/docs/ReleaseNotes.rst.rej
@@ -0,0 +1,24 @@
+diff a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst (rejected hunks)
+@@ -404,6 +404,12 @@ Attribute Changes in Clang
+ format string must correctly format the fixed parameter types of the function.
+ Using the attribute this way emits a GCC compatibility diagnostic.
+
++- Support was added for ``__attribute__((function_return("thunk-extern")))``
++ to X86 to replace ``ret`` instructions with ``jmp __x86_return_thunk``. The
++ corresponding attribute to disable this,
++ ``__attribute__((function_return("keep")))`` was added. This is intended to
++ be used by the Linux kernel to mitigate RETBLEED.
++
+ Windows Support
+ ---------------
+
+@@ -556,6 +562,9 @@ X86 Support in Clang
+ this instruction (see rdpruintrin.h).
+ - Support ``-mstack-protector-guard-symbol=[SymbolName]`` to use the given
+ symbol for addressing the stack protector guard.
++- ``-mfunction-return=thunk-extern`` support was added to clang for x86. This
++ will be used by Linux kernel mitigations for RETBLEED. The corresponding flag
++ ``-mfunction-return=keep`` may be appended to disable the feature.
+
+ DWARF Support in Clang
+ ----------------------
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index c7141ff463b3..f3c9a511e189 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -3957,3 +3957,14 @@ def HLSLNumThreads: InheritableAttr {
let LangOpts = [HLSL];
let Documentation = [NumThreadsDocs];
}
+
+def FunctionReturnThunks : InheritableAttr,
+ TargetSpecificAttr<TargetAnyX86> {
+ let Spellings = [GCC<"function_return">];
+ let Args = [EnumArgument<"ThunkType", "Kind",
+ ["keep", "thunk-extern"],
+ ["Keep", "Extern"]
+ >];
+ let Subjects = SubjectList<[Function]>;
+ let Documentation = [FunctionReturnThunksDocs];
+}
diff --git a/clang/include/clang/Basic/Attr.td.rej b/clang/include/clang/Basic/Attr.td.rej
new file mode 100644
index 000000000000..0874ce670a59
--- /dev/null
+++ b/clang/include/clang/Basic/Attr.td.rej
@@ -0,0 +1,16 @@
+diff a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td (rejected hunks)
+@@ -4036,3 +4036,14 @@ def NoRandomizeLayout : InheritableAttr {
+ let LangOpts = [COnly];
+ }
+ def : MutualExclusions<[RandomizeLayout, NoRandomizeLayout]>;
++
++def FunctionReturnThunks : InheritableAttr,
++ TargetSpecificAttr<TargetAnyX86> {
++ let Spellings = [GCC<"function_return">];
++ let Args = [EnumArgument<"ThunkType", "Kind",
++ ["keep", "thunk-extern"],
++ ["Keep", "Extern"]
++ >];
++ let Subjects = SubjectList<[Function]>;
++ let Documentation = [FunctionReturnThunksDocs];
++}
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 89db454f7dac..983a394cd4b2 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -6379,3 +6379,25 @@ dictate the thread id. Total number of threads executed is ``X * Y * Z``.
The full documentation is available here: https://docs.microsoft.com/en-us/windows/win32/direct3dhlsl/sm5-attributes-numthreads
}];
}
+
+def FunctionReturnThunksDocs : Documentation {
+ let Category = DocCatFunction;
+ let Content = [{
+The attribute ``function_return`` can replace return instructions with jumps to
+target-specific symbols. This attribute supports 2 possible values,
+corresponding to the values supported by the ``-mfunction-return=`` command
+line flag:
+* ``__attribute__((function_return("keep")))`` to disable related transforms.
+ This is useful for undoing global setting from ``-mfunction-return=`` locally
+ for individual functions.
+* ``__attribute__((function_return("thunk-extern")))`` to replace returns with
+ jumps, while NOT emitting the thunk.
+
+The values ``thunk`` and ``thunk-inline`` from GCC are not supported.
+
+The symbol used for ``thunk-extern`` is target specific:
+* X86: ``__x86_return_thunk``
+
+As such, this function attribute is currently only supported on X86 targets.
+ }];
+ }
diff --git a/clang/include/clang/Basic/AttrDocs.td.rej b/clang/include/clang/Basic/AttrDocs.td.rej
new file mode 100644
index 000000000000..d431db7d9a38
--- /dev/null
+++ b/clang/include/clang/Basic/AttrDocs.td.rej
@@ -0,0 +1,29 @@
+diff a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td (rejected hunks)
+@@ -6606,6 +6606,27 @@ evaluate to NULL.
+ }
+ return 0;
+ }
++ }];
++}
++
++def FunctionReturnThunksDocs : Documentation {
++ let Category = DocCatFunction;
++ let Content = [{
++The attribute ``function_return`` can replace return instructions with jumps to
++target-specific symbols. This attribute supports 2 possible values,
++corresponding to the values supported by the ``-mfunction-return=`` command
++line flag:
++* ``__attribute__((function_return("keep")))`` to disable related transforms.
++ This is useful for undoing global setting from ``-mfunction-return=`` locally
++ for individual functions.
++* ``__attribute__((function_return("thunk-extern")))`` to replace returns with
++ jumps, while NOT emitting the thunk.
++
++The values ``thunk`` and ``thunk-inline`` from GCC are not supported.
++
++The symbol used for ``thunk-extern`` is target specific:
++* X86: ``__x86_return_thunk``
+
++As such, this function attribute is currently only supported on X86 targets.
+ }];
+ }
diff --git a/clang/include/clang/Basic/CodeGenOptions.def b/clang/include/clang/Basic/CodeGenOptions.def
index 720a59ff1bdc..8a3af1faeaf1 100644
--- a/clang/include/clang/Basic/CodeGenOptions.def
+++ b/clang/include/clang/Basic/CodeGenOptions.def
@@ -108,6 +108,7 @@ CODEGENOPT(CFProtectionReturn , 1, 0) ///< if -fcf-protection is
CODEGENOPT(CFProtectionBranch , 1, 0) ///< if -fcf-protection is
///< set to full or branch.
CODEGENOPT(IBTSeal, 1, 0) ///< set to optimize CFProtectionBranch.
+CODEGENOPT(FunctionReturnThunks, 1, 0) ///< -mfunction-return={keep|thunk-extern}
CODEGENOPT(XRayInstrumentFunctions , 1, 0) ///< Set when -fxray-instrument is
///< enabled.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 799fd1b62f18..9b6c04f94dd3 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1967,6 +1967,13 @@ def fcf_protection : Flag<["-"], "fcf-protection">, Group<f_Group>, Flags<[CoreO
HelpText<"Enable cf-protection in 'full' mode">;
def mibt_seal : Flag<["-"], "mibt-seal">, Group<m_Group>, Flags<[CoreOption, CC1Option]>,
HelpText<"Optimize fcf-protection=branch/full (requires LTO).">;
+def mfunction_return_EQ : Joined<["-"], "mfunction-return=">,
+ Group<m_Group>, Flags<[CoreOption, CC1Option]>,
+ HelpText<"Replace returns with jumps to ``__x86_return_thunk`` (x86 only, error otherwise)">,
+ Values<"keep,thunk-extern">,
+ NormalizedValues<["Keep", "Extern"]>,
+ NormalizedValuesScope<"llvm::FunctionReturnThunksKind">,
+ MarshallingInfoEnum<CodeGenOpts<"FunctionReturnThunks">, "Keep">;
defm xray_instrument : BoolFOption<"xray-instrument",
LangOpts<"XRayInstrument">, DefaultFalse,
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 32d329a69b9f..cfcfa3d8b5c7 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -920,6 +920,20 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
if (D && D->hasAttr<NoProfileFunctionAttr>())
Fn->addFnAttr(llvm::Attribute::NoProfile);
+ if (D) {
+ // Function attributes take precedence over command line flags.
+ if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
+ switch (A->getThunkType()) {
+ case FunctionReturnThunksAttr::Kind::Keep:
+ break;
+ case FunctionReturnThunksAttr::Kind::Extern:
+ Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);
+ break;
+ }
+ } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
+ Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);
+ }
+
if (FD && getLangOpts().OpenCL) {
// Add metadata for a kernel function.
EmitOpenCLKernelMetadata(FD, Fn);
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp.rej b/clang/lib/CodeGen/CodeGenFunction.cpp.rej
new file mode 100644
index 000000000000..4e0388ca4a06
--- /dev/null
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp.rej
@@ -0,0 +1,22 @@
+diff a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp (rejected hunks)
+@@ -898,6 +898,20 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
+ if (D && D->hasAttr<NoProfileFunctionAttr>())
+ Fn->addFnAttr(llvm::Attribute::NoProfile);
+
++ if (D) {
++ // Function attributes take precedence over command line flags.
++ if (auto *A = D->getAttr<FunctionReturnThunksAttr>()) {
++ switch (A->getThunkType()) {
++ case FunctionReturnThunksAttr::Kind::Keep:
++ break;
++ case FunctionReturnThunksAttr::Kind::Extern:
++ Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);
++ break;
++ }
++ } else if (CGM.getCodeGenOpts().FunctionReturnThunks)
++ Fn->addFnAttr(llvm::Attribute::FnRetThunkExtern);
++ }
++
+ if (FD && (getLangOpts().OpenCL ||
+ (getLangOpts().HIP && getLangOpts().CUDAIsDevice))) {
+ // Add metadata for a kernel function.
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index baac938412ec..e5d73db7f7d3 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -6268,6 +6268,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
if (IsUsingLTO)
Args.AddLastArg(CmdArgs, options::OPT_mibt_seal);
+ if (Arg *A = Args.getLastArg(options::OPT_mfunction_return_EQ))
+ CmdArgs.push_back(
+ Args.MakeArgString(Twine("-mfunction-return=") + A->getValue()));
+
// Forward -f options with positive and negative forms; we translate these by
// hand. Do not propagate PGO options to the GPU-side compilations as the
// profile info is for the host-side compilation only.
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 83de27b3a4f1..c7b7f54c97b6 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1494,6 +1494,9 @@ void CompilerInvocation::GenerateCodeGenArgs(
else if (Opts.CFProtectionBranch)
GenerateArg(Args, OPT_fcf_protection_EQ, "branch", SA);
+ if (Opts.FunctionReturnThunks)
+ GenerateArg(Args, OPT_mfunction_return_EQ, "thunk-extern", SA);
+
for (const auto &F : Opts.LinkBitcodeFiles) {
bool Builtint = F.LinkFlags == llvm::Linker::Flags::LinkOnlyNeeded &&
F.PropagateAttrs && F.Internalize;
@@ -1835,6 +1838,27 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
}
+ if (const Arg *A = Args.getLastArg(OPT_mfunction_return_EQ)) {
+ auto Val = llvm::StringSwitch<llvm::FunctionReturnThunksKind>(A->getValue())
+ .Case("keep", llvm::FunctionReturnThunksKind::Keep)
+ .Case("thunk-extern", llvm::FunctionReturnThunksKind::Extern)
+ .Default(llvm::FunctionReturnThunksKind::Invalid);
+ // SystemZ might want to add support for "expolines."
+ if (!T.isX86())
+ Diags.Report(diag::err_drv_argument_not_allowed_with)
+ << A->getSpelling() << T.getTriple();
+ else if (Val == llvm::FunctionReturnThunksKind::Invalid)
+ Diags.Report(diag::err_drv_invalid_value)
+ << A->getAsString(Args) << A->getValue();
+ else if (Val == llvm::FunctionReturnThunksKind::Extern &&
+ Args.getLastArgValue(OPT_mcmodel_EQ).equals("large"))
+ Diags.Report(diag::err_drv_argument_not_allowed_with)
+ << A->getAsString(Args)
+ << Args.getLastArg(OPT_mcmodel_EQ)->getAsString(Args);
+ else
+ Opts.FunctionReturnThunks = static_cast<unsigned>(Val);
+ }
+
if (Opts.PrepareForLTO && Args.hasArg(OPT_mibt_seal))
Opts.IBTSeal = 1;
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp.rej b/clang/lib/Frontend/CompilerInvocation.cpp.rej
new file mode 100644
index 000000000000..92b8471ce545
--- /dev/null
+++ b/clang/lib/Frontend/CompilerInvocation.cpp.rej
@@ -0,0 +1,11 @@
+diff a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp (rejected hunks)
+@@ -1485,6 +1485,9 @@ void CompilerInvocation::GenerateCodeGenArgs(
+ if (Opts.IBTSeal)
+ GenerateArg(Args, OPT_mibt_seal, SA);
+
++ if (Opts.FunctionReturnThunks)
++ GenerateArg(Args, OPT_mfunction_return_EQ, "thunk-extern", SA);
++
+ for (const auto &F : Opts.LinkBitcodeFiles) {
+ bool Builtint = F.LinkFlags == llvm::Linker::Flags::LinkOnlyNeeded &&
+ F.PropagateAttrs && F.Internalize;
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 4b5201db7517..59adebe91d1f 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -7967,6 +7967,26 @@ static void handleZeroCallUsedRegsAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
D->addAttr(ZeroCallUsedRegsAttr::Create(S.Context, Kind, AL));
}
+static void handleFunctionReturnThunksAttr(Sema &S, Decl *D,
+ const ParsedAttr &AL) {
+ StringRef KindStr;
+ SourceLocation LiteralLoc;
+ if (!S.checkStringLiteralArgumentAttr(AL, 0, KindStr, &LiteralLoc))
+ return;
+
+ FunctionReturnThunksAttr::Kind Kind;
+ if (!FunctionReturnThunksAttr::ConvertStrToKind(KindStr, Kind)) {
+ S.Diag(LiteralLoc, diag::warn_attribute_type_not_supported)
+ << AL << KindStr;
+ return;
+ }
+ // FIXME: it would be good to better handle attribute merging rather than
+ // silently replacing the existing attribute, so long as it does not break
+ // the expected codegen tests.
+ D->dropAttr<FunctionReturnThunksAttr>();
+ D->addAttr(FunctionReturnThunksAttr::Create(S.Context, Kind, AL));
+}
+
static void handleSYCLKernelAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
// The 'sycl_kernel' attribute applies only to function templates.
const auto *FD = cast<FunctionDecl>(D);
@@ -8757,6 +8777,9 @@ static void ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D,
case ParsedAttr::AT_ZeroCallUsedRegs:
handleZeroCallUsedRegsAttr(S, D, AL);
break;
+ case ParsedAttr::AT_FunctionReturnThunks:
+ handleFunctionReturnThunksAttr(S, D, AL);
+ break;
// Microsoft attributes:
case ParsedAttr::AT_LayoutVersion:
diff --git a/clang/test/CodeGen/attr-function-return.c b/clang/test/CodeGen/attr-function-return.c
new file mode 100644
index 000000000000..8b68cfe52afa
--- /dev/null
+++ b/clang/test/CodeGen/attr-function-return.c
@@ -0,0 +1,96 @@
+// RUN: %clang_cc1 -std=gnu2x -triple x86_64-linux-gnu %s -emit-llvm -o - \
+// RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-NOM
+// RUN: %clang_cc1 -std=gnu2x -triple x86_64-linux-gnu %s -emit-llvm -o - \
+// RUN: -mfunction-return=keep | FileCheck %s \
+// RUN: --check-prefixes=CHECK,CHECK-KEEP
+// RUN: %clang_cc1 -std=gnu2x -triple x86_64-linux-gnu %s -emit-llvm -o - \
+// RUN: -mfunction-return=thunk-extern | FileCheck %s \
+// RUN: --check-prefixes=CHECK,CHECK-EXTERN
+
+#if !__has_attribute(function_return)
+#error "missing attribute support for function_return"
+#endif
+
+// CHECK: @keep() [[KEEP:#[0-9]+]]
+__attribute__((function_return("keep"))) void keep(void) {}
+
+// CHECK: @keep2() [[KEEP:#[0-9]+]]
+[[gnu::function_return("keep")]] void keep2(void) {}
+
+// CHECK: @thunk_extern() [[EXTERN:#[0-9]+]]
+__attribute__((function_return("thunk-extern"))) void thunk_extern(void) {}
+
+// CHECK: @thunk_extern2() [[EXTERN:#[0-9]+]]
+[[gnu::function_return("thunk-extern")]] void thunk_extern2(void) {}
+
+// CHECK: @double_thunk_keep() [[KEEP]]
+// clang-format off
+__attribute__((function_return("thunk-extern")))
+__attribute__((function_return("keep")))
+void double_thunk_keep(void) {}
+
+// CHECK: @double_thunk_keep2() [[KEEP]]
+[[gnu::function_return("thunk-extern")]][[gnu::function_return("keep")]]
+void double_thunk_keep2(void) {}
+
+// CHECK: @double_keep_thunk() [[EXTERN]]
+__attribute__((function_return("keep")))
+__attribute__((function_return("thunk-extern")))
+void double_keep_thunk(void) {}
+
+// CHECK: @double_keep_thunk2() [[EXTERN]]
+[[gnu::function_return("thunk-keep")]][[gnu::function_return("thunk-extern")]]
+void double_keep_thunk2(void) {}
+
+// CHECK: @thunk_keep() [[KEEP]]
+__attribute__((function_return("thunk-extern"), function_return("keep")))
+void thunk_keep(void) {}
+
+// CHECK: @thunk_keep2() [[KEEP]]
+[[gnu::function_return("thunk-extern"), gnu::function_return("keep")]]
+void thunk_keep2(void) {}
+
+// CHECK: @keep_thunk() [[EXTERN]]
+__attribute__((function_return("keep"), function_return("thunk-extern")))
+void keep_thunk(void) {}
+
+// CHECK: @keep_thunk2() [[EXTERN]]
+[[gnu::function_return("keep"), gnu::function_return("thunk-extern")]]
+void keep_thunk2(void) {}
+// clang-format on
+
+void undef(void);
+// CHECK: @undef() [[KEEP]]
+__attribute__((function_return("keep"))) void undef(void) {}
+
+void undef2(void);
+// CHECK: @undef2() [[EXTERN]]
+__attribute__((function_return("thunk-extern"))) void undef2(void) {}
+
+__attribute__((function_return("thunk-extern"))) void change_def(void);
+// CHECK: @change_def() [[KEEP]]
+__attribute__((function_return("keep"))) void change_def(void) {}
+
+__attribute__((function_return("keep"))) void change_def2(void);
+// CHECK: @change_def2() [[EXTERN]]
+__attribute__((function_return("thunk-extern"))) void change_def2(void) {}
+
+__attribute__((function_return("thunk-extern"))) void change_def3(void);
+// CHECK: @change_def3() [[KEEP]]
+[[gnu::function_return("keep")]] void change_def3(void) {}
+
+[[gnu::function_return("keep")]] void change_def4(void);
+// CHECK: @change_def4() [[EXTERN]]
+__attribute__((function_return("thunk-extern"))) void change_def4(void) {}
+
+// When there is no -mfunction-return= flag set (NOM) or it's set to keep,
+// we don't emit anything into the IR for unattributed functions.
+
+// CHECK-NOM: @no_attrs() [[NOATTR:#[0-9]+]]
+// CHECK-KEEP: @no_attrs() [[NOATTR:#[0-9]+]]
+// CHECK-EXTERN: @no_attrs() [[EXTERN]]
+void no_attrs(void) {}
+
+// CHECK-NOM-NOT: [[NOATTR]] = {{.*}}fn_ret_thunk_extern
+// CHECK-KEEP-NOT: [[NOATTR]] = {{.*}}fn_ret_thunk_extern
+// CHECK: [[EXTERN]] = {{.*}}fn_ret_thunk_extern
diff --git a/clang/test/CodeGen/attr-function-return.cpp b/clang/test/CodeGen/attr-function-return.cpp
new file mode 100644
index 000000000000..9d58b7b2f857
--- /dev/null
+++ b/clang/test/CodeGen/attr-function-return.cpp
@@ -0,0 +1,52 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -emit-llvm -o - \
+// RUN: | FileCheck %s --check-prefixes=CHECK,CHECK-NOM
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -emit-llvm -o - \
+// RUN: -mfunction-return=keep | FileCheck %s \
+// RUN: --check-prefixes=CHECK,CHECK-KEEP
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -emit-llvm -o - \
+// RUN: -mfunction-return=thunk-extern | FileCheck %s \
+// RUN: --check-prefixes=CHECK,CHECK-EXTERN
+
+int foo(void) {
+ // CHECK: @"_ZZ3foovENK3$_0clEv"({{.*}}) [[NOATTR:#[0-9]+]]
+ return []() {
+ return 42;
+ }();
+}
+int bar(void) {
+ // CHECK: @"_ZZ3barvENK3$_1clEv"({{.*}}) [[EXTERN:#[0-9]+]]
+ return []() __attribute__((function_return("thunk-extern"))) {
+ return 42;
+ }
+ ();
+}
+int baz(void) {
+ // CHECK: @"_ZZ3bazvENK3$_2clEv"({{.*}}) [[KEEP:#[0-9]+]]
+ return []() __attribute__((function_return("keep"))) {
+ return 42;
+ }
+ ();
+}
+
+class Foo {
+public:
+ // CHECK: @_ZN3Foo3fooEv({{.*}}) [[EXTERN]]
+ __attribute__((function_return("thunk-extern"))) int foo() { return 42; }
+};
+
+int quux() {
+ Foo my_foo;
+ return my_foo.foo();
+}
+
+// CHECK: @extern_c() [[EXTERN]]
+extern "C" __attribute__((function_return("thunk-extern"))) void extern_c() {}
+extern "C" {
+// CHECK: @extern_c2() [[EXTERN]]
+__attribute__((function_return("thunk-extern"))) void extern_c2() {}
+}
+
+// CHECK-NOM-NOT: [[NOATTR]] = {{.*}}fn_ret_thunk_extern
+// CHECK-KEEP-NOT: [[NOATTR]] = {{.*}}fn_ret_thunk_extern
+// CHECK-KEEP-NOT: [[KEEP]] = {{.*}}fn_ret_thunk_extern
+// CHECK-EXTERN: [[EXTERN]] = {{.*}}fn_ret_thunk_extern
diff --git a/clang/test/Driver/mfunction-return.c b/clang/test/Driver/mfunction-return.c
new file mode 100644
index 000000000000..fca010aefa55
--- /dev/null
+++ b/clang/test/Driver/mfunction-return.c
@@ -0,0 +1,22 @@
+// RUN: %clang -mfunction-return= -### %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-VALID %s
+// RUN: not %clang -mfunction-return -### %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-INVALID %s
+
+// RUN: %clang -mfunction-return=keep -### %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-KEEP %s
+// RUN: %clang -mfunction-return=thunk-extern -### %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-EXTERN %s
+
+// RUN: %clang -mfunction-return=keep -mfunction-return=thunk-extern -### %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-EXTERN %s
+// RUN: %clang -mfunction-return=thunk-extern -mfunction-return=keep -### %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-KEEP %s
+
+// CHECK-VALID: "-mfunction-return="
+// CHECK-INVALID: error: unknown argument: '-mfunction-return'
+
+// CHECK-KEEP: "-mfunction-return=keep"
+// CHECK-KEEP-NOT: "-mfunction-return=thunk-extern"
+// CHECK-EXTERN: "-mfunction-return=thunk-extern"
+// CHECK-EXTERN-NOT: "-mfunction-return=keep"
diff --git a/clang/test/Frontend/mfunction-return.c b/clang/test/Frontend/mfunction-return.c
new file mode 100644
index 000000000000..9aefa0f8c0a8
--- /dev/null
+++ b/clang/test/Frontend/mfunction-return.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -mfunction-return=keep -triple x86_64-linux-gnu %s
+// RUN: %clang_cc1 -mfunction-return=thunk-extern -triple x86_64-linux-gnu %s
+
+// RUN: not %clang_cc1 -mfunction-return=thunk -triple x86_64-linux-gnu %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-THUNK %s
+// RUN: not %clang_cc1 -mfunction-return=thunk-inline -triple x86_64-linux-gnu %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-INLINE %s
+// RUN: not %clang_cc1 -mfunction-return=invalid -triple x86_64-linux-gnu %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-INVALID %s
+// RUN: not %clang_cc1 -mfunction-return=thunk-extern -triple s390x-linux-gnu %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-TARGET %s
+// RUN: not %clang_cc1 -mfunction-return=thunk-extern -mcmodel=large \
+// RUN: -triple x86_64-linux-gnu %s 2>&1 \
+// RUN: | FileCheck --check-prefix=CHECK-LARGE %s
+
+// CHECK-THUNK: error: invalid value 'thunk' in '-mfunction-return=thunk'
+// CHECK-INLINE: error: invalid value 'thunk-inline' in '-mfunction-return=thunk-inline'
+// CHECK-INVALID: error: invalid value 'invalid' in '-mfunction-return=invalid'
+// CHECK-TARGET: error: invalid argument '-mfunction-return=' not allowed with 's390x-unknown-linux-gnu'
+// CHECK-LARGE: error: invalid argument '-mfunction-return=thunk-extern' not allowed with '-mcmodel=large'
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index daae71f5167c..93cb025fa63f 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -69,6 +69,7 @@
// CHECK-NEXT: ExternalSourceSymbol ((SubjectMatchRule_record, SubjectMatchRule_enum, SubjectMatchRule_enum_constant, SubjectMatchRule_field, SubjectMatchRule_function, SubjectMatchRule_namespace, SubjectMatchRule_objc_category, SubjectMatchRule_objc_implementation, SubjectMatchRule_objc_interface, SubjectMatchRule_objc_method, SubjectMatchRule_objc_property, SubjectMatchRule_objc_protocol, SubjectMatchRule_record, SubjectMatchRule_type_alias, SubjectMatchRule_variable))
// CHECK-NEXT: FlagEnum (SubjectMatchRule_enum)
// CHECK-NEXT: Flatten (SubjectMatchRule_function)
+// CHECK-NEXT: FunctionReturnThunks (SubjectMatchRule_function)
// CHECK-NEXT: GNUInline (SubjectMatchRule_function)
// CHECK-NEXT: HIPManaged (SubjectMatchRule_variable)
// CHECK-NEXT: Hot (SubjectMatchRule_function)
diff --git a/clang/test/Sema/attr-function-return-unsupported-target.c b/clang/test/Sema/attr-function-return-unsupported-target.c
new file mode 100644
index 000000000000..e00eecc50ad9
--- /dev/null
+++ b/clang/test/Sema/attr-function-return-unsupported-target.c
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -triple s390x-linux-gnu -fsyntax-only -verify %s
+
+// expected-warning@+1 {{unknown attribute 'function_return' ignored}}
+__attribute__((function_return("keep"))) void x(void) {}
+
+// expected-warning@+1 {{unknown attribute 'function_return' ignored}}
+__attribute__((function_return("thunk"))) void y(void) {}
+
+// expected-warning@+1 {{unknown attribute 'function_return' ignored}}
+__attribute__((function_return("thunk-inline"))) void z(void) {}
+
+// expected-warning@+1 {{unknown attribute 'function_return' ignored}}
+__attribute__((function_return("thunk-extern"))) void w(void) {}
+
+// expected-warning@+1 {{unknown attribute 'function_return' ignored}}
+__attribute__((function_return("invalid"))) void v(void) {}
diff --git a/clang/test/Sema/attr-function-return.c b/clang/test/Sema/attr-function-return.c
new file mode 100644
index 000000000000..c6fe88b821e3
--- /dev/null
+++ b/clang/test/Sema/attr-function-return.c
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsyntax-only -verify %s
+
+__attribute__((function_return("keep"))) void x(void) {}
+
+// expected-warning@+1 {{'function_return' attribute argument not supported: thunk}}
+__attribute__((function_return("thunk"))) void y(void) {}
+
+// expected-warning@+1 {{'function_return' attribute argument not supported: thunk-inline}}
+__attribute__((function_return("thunk-inline"))) void z(void) {}
+
+__attribute__((function_return("thunk-extern"))) void w(void) {}
+
+// expected-warning@+1 {{'function_return' attribute argument not supported: invalid}}
+__attribute__((function_return("invalid"))) void v(void) {}
+
+// expected-error@+1 {{'function_return' attribute requires a string}}
+__attribute__((function_return(5))) void a(void) {}
+
+// expected-error@+1 {{'function_return' attribute takes one argument}}
+__attribute__((function_return)) void b(void) {}
+
+// expected-warning@+1 {{'function_return' attribute only applies to functions}}
+__attribute__((function_return)) int c;
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 1a218c0a5c75..f1309f0626b7 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1625,6 +1625,10 @@ example:
Front ends can provide optional ``srcloc`` metadata nodes on call sites of
such callees to attach information about where in the source language such a
call came from. A string value can be provided as a note.
+``fn_ret_thunk_extern``
+ This attribute tells the code generator that returns from functions should
+ be replaced with jumps to externally-defined architecture-specific symbols.
+ For X86, this symbol's identifier is ``__x86_return_thunk``.
``"frame-pointer"``
This attribute tells the code generator whether the function
should keep the frame pointer. The code generator may emit the frame pointer
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 2abb306f89d5..f7c5009ce646 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -682,6 +682,7 @@ enum AttributeKindCodes {
ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION = 78,
ATTR_KIND_NO_SANITIZE_BOUNDS = 79,
ATTR_KIND_ALLOC_ALIGN = 80,
+ ATTR_KIND_FNRETTHUNK_EXTERN = 81,
};
enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h.rej b/llvm/include/llvm/Bitcode/LLVMBitCodes.h.rej
new file mode 100644
index 000000000000..7baa1939301e
--- /dev/null
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h.rej
@@ -0,0 +1,9 @@
+diff a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h (rejected hunks)
+@@ -688,6 +688,7 @@ enum AttributeKindCodes {
+ ATTR_KIND_ALLOCATED_POINTER = 81,
+ ATTR_KIND_ALLOC_KIND = 82,
+ ATTR_KIND_PRESPLIT_COROUTINE = 83,
++ ATTR_KIND_FNRETTHUNK_EXTERN = 84,
+ };
+
+ enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index 66212a7da8b3..d2d877aff8e4 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -96,6 +96,10 @@ def DisableSanitizerInstrumentation: EnumAttr<"disable_sanitizer_instrumentation
/// Provide pointer element type to intrinsic.
def ElementType : TypeAttr<"elementtype", [ParamAttr]>;
+/// Whether to keep return instructions, or replace with a jump to an external
+/// symbol.
+def FnRetThunkExtern : EnumAttr<"fn_ret_thunk_extern", [FnAttr]>;
+
/// Function may only access memory that is inaccessible from IR.
def InaccessibleMemOnly : EnumAttr<"inaccessiblememonly", [FnAttr]>;
diff --git a/llvm/include/llvm/Support/CodeGen.h b/llvm/include/llvm/Support/CodeGen.h
index 71d0ddbfe05e..425d3a3d95d4 100644
--- a/llvm/include/llvm/Support/CodeGen.h
+++ b/llvm/include/llvm/Support/CodeGen.h
@@ -103,6 +103,13 @@ namespace llvm {
Async = 2, ///< "Asynchronous" unwind tables (instr precise)
Default = 2,
};
+
+ enum class FunctionReturnThunksKind : unsigned int {
+ Keep = 0, ///< No function return thunk.
+ Extern = 1, ///< Replace returns with jump to thunk, don't emit thunk.
+ Invalid = 2, ///< Not used.
+ };
+
} // namespace llvm
#endif
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 8a5074941b97..ec3b7e4369cb 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1478,6 +1478,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
return Attribute::DisableSanitizerInstrumentation;
case bitc::ATTR_KIND_ELEMENTTYPE:
return Attribute::ElementType;
+ case bitc::ATTR_KIND_FNRETTHUNK_EXTERN:
+ return Attribute::FnRetThunkExtern;
case bitc::ATTR_KIND_INACCESSIBLEMEM_ONLY:
return Attribute::InaccessibleMemOnly;
case bitc::ATTR_KIND_INACCESSIBLEMEM_OR_ARGMEMONLY:
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index c76294d5a6d9..74d62c6200e5 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -630,6 +630,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
return bitc::ATTR_KIND_COLD;
case Attribute::DisableSanitizerInstrumentation:
return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
+ case Attribute::FnRetThunkExtern:
+ return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
case Attribute::Hot:
return bitc::ATTR_KIND_HOT;
case Attribute::ElementType:
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index 5a1d04e2d835..2327b73bf608 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -73,6 +73,7 @@ set(sources
X86PartialReduction.cpp
X86RegisterBankInfo.cpp
X86RegisterInfo.cpp
+ X86ReturnThunks.cpp
X86SelectionDAGInfo.cpp
X86ShuffleDecodeConstantPool.cpp
X86SpeculativeLoadHardening.cpp
diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h
index 10e1c5d6ed38..453e10a5b72f 100644
--- a/llvm/lib/Target/X86/X86.h
+++ b/llvm/lib/Target/X86/X86.h
@@ -129,6 +129,9 @@ FunctionPass *createX86EvexToVexInsts();
/// This pass creates the thunks for the retpoline feature.
FunctionPass *createX86IndirectThunksPass();
+/// This pass replaces ret instructions with jmp's to __x86_return thunk.
+FunctionPass *createX86ReturnThunksPass();
+
/// This pass ensures instructions featuring a memory operand
/// have distinctive <LineNumber, Discriminator> (with respect to eachother)
FunctionPass *createX86DiscriminateMemOpsPass();
@@ -181,6 +184,7 @@ void initializeX86LowerAMXTypeLegacyPassPass(PassRegistry &);
void initializeX86PreAMXConfigPassPass(PassRegistry &);
void initializeX86LowerTileCopyPass(PassRegistry &);
void initializeX86LowerAMXIntrinsicsLegacyPassPass(PassRegistry &);
+void initializeX86ReturnThunksPass(PassRegistry &);
namespace X86AS {
enum : unsigned {
diff --git a/llvm/lib/Target/X86/X86ReturnThunks.cpp b/llvm/lib/Target/X86/X86ReturnThunks.cpp
new file mode 100644
index 000000000000..4b203229ba83
--- /dev/null
+++ b/llvm/lib/Target/X86/X86ReturnThunks.cpp
@@ -0,0 +1,92 @@
+//==- X86ReturnThunks.cpp - Replace rets with thunks or inline thunks --=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// Pass that replaces ret instructions with a jmp to __x86_return_thunk.
+///
+/// This corresponds to -mfunction-return=thunk-extern or
+/// __attribute__((function_return("thunk-extern").
+///
+/// This pass is a minimal implementation necessary to help mitigate
+/// RetBleed for the Linux kernel.
+///
+/// Should support for thunk or thunk-inline be necessary in the future, then
+/// this pass should be combined with x86-retpoline-thunks which already has
+/// machinery to emit thunks. Until then, YAGNI.
+///
+/// This pass is very similar to x86-lvi-ret.
+///
+//===----------------------------------------------------------------------===//
+
+#include "X86.h"
+#include "X86InstrInfo.h"
+#include "X86Subtarget.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/MC/MCInstrDesc.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define PASS_KEY "x86-return-thunks"
+#define DEBUG_TYPE PASS_KEY
+
+struct X86ReturnThunks final : public MachineFunctionPass {
+ static char ID;
+ X86ReturnThunks() : MachineFunctionPass(ID) {}
+ StringRef getPassName() const override { return "X86 Return Thunks"; }
+ bool runOnMachineFunction(MachineFunction &MF) override;
+};
+
+char X86ReturnThunks::ID = 0;
+
+bool X86ReturnThunks::runOnMachineFunction(MachineFunction &MF) {
+ LLVM_DEBUG(dbgs() << getPassName() << "\n");
+
+ bool Modified = false;
+
+ if (!MF.getFunction().hasFnAttribute(llvm::Attribute::FnRetThunkExtern))
+ return Modified;
+
+ StringRef ThunkName = "__x86_return_thunk";
+ if (MF.getFunction().getName() == ThunkName)
+ return Modified;
+
+ const auto &ST = MF.getSubtarget<X86Subtarget>();
+ const bool Is64Bit = ST.getTargetTriple().getArch() == Triple::x86_64;
+ const unsigned RetOpc = Is64Bit ? X86::RET64 : X86::RET32;
+ SmallVector<MachineInstr *, 16> Rets;
+
+ for (MachineBasicBlock &MBB : MF)
+ for (MachineInstr &Term : MBB.terminators())
+ if (Term.getOpcode() == RetOpc)
+ Rets.push_back(&Term);
+
+ const MCInstrDesc &JMP = ST.getInstrInfo()->get(X86::TAILJMPd);
+
+ for (MachineInstr *Ret : Rets) {
+ BuildMI(Ret->getParent(), Ret->getDebugLoc(), JMP)
+ .addExternalSymbol(ThunkName.data());
+ Ret->eraseFromParent();
+ Modified = true;
+ }
+
+ return Modified;
+}
+
+INITIALIZE_PASS(X86ReturnThunks, PASS_KEY, "X86 Return Thunks", false, false)
+
+FunctionPass *llvm::createX86ReturnThunksPass() {
+ return new X86ReturnThunks();
+}
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index aa44e627bb1f..d88ac4fba0a8 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -92,6 +92,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeX86Target() {
initializeX86OptimizeLEAPassPass(PR);
initializeX86PartialReductionPass(PR);
initializePseudoProbeInserterPass(PR);
+ initializeX86ReturnThunksPass(PR);
}
static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -569,6 +570,7 @@ void X86PassConfig::addPreEmitPass2() {
// hand inspection of the codegen output.
addPass(createX86SpeculativeExecutionSideEffectSuppression());
addPass(createX86IndirectThunksPass());
+ addPass(createX86ReturnThunksPass());
// Insert extra int3 instructions after trailing call instructions to avoid
// issues in the unwinder.
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 37689225b646..d9dc48548e7c 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -925,6 +925,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
case Attribute::AlwaysInline:
case Attribute::Cold:
case Attribute::DisableSanitizerInstrumentation:
+ case Attribute::FnRetThunkExtern:
case Attribute::Hot:
case Attribute::NoRecurse:
case Attribute::InlineHint:
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index ee582cab628d..78b54a5cf5c3 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -532,6 +532,9 @@ define void @f86() nosanitize_bounds
ret void;
}
+; CHECK: define void @f87() [[FNRETTHUNKEXTERN:#[0-9]+]]
+define void @f87() fn_ret_thunk_extern { ret void }
+
; CHECK: attributes #0 = { noreturn }
; CHECK: attributes #1 = { nounwind }
; CHECK: attributes #2 = { readnone }
@@ -585,4 +588,5 @@ define void @f86() nosanitize_bounds
; CHECK: attributes #50 = { disable_sanitizer_instrumentation }
; CHECK: attributes #51 = { uwtable(sync) }
; CHECK: attributes #52 = { nosanitize_bounds }
+; CHECK: attributes [[FNRETTHUNKEXTERN]] = { fn_ret_thunk_extern }
; CHECK: attributes #[[NOBUILTIN]] = { nobuiltin }
diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll
index 54eecb113540..12740c361a01 100644
--- a/llvm/test/CodeGen/X86/O0-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O0-pipeline.ll
@@ -71,8 +71,9 @@
; CHECK-NEXT: Live DEBUG_VALUE analysis
; CHECK-NEXT: X86 Speculative Execution Side Effect Suppression
; CHECK-NEXT: X86 Indirect Thunks
+; CHECK-NEXT: X86 Return Thunks
; CHECK-NEXT: Check CFA info and insert CFI instructions if needed
-; CHECK-NEXT: X86 Load Value Injection (LVI) Ret-Hardening
+; CHECK-NEXT: X86 Load Value Injection (LVI) Ret-Hardening
; CHECK-NEXT: Pseudo Probe Inserter
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine Optimization Remark Emitter
diff --git a/llvm/test/CodeGen/X86/attr-function-return.ll b/llvm/test/CodeGen/X86/attr-function-return.ll
new file mode 100644
index 000000000000..f40d971ed355
--- /dev/null
+++ b/llvm/test/CodeGen/X86/attr-function-return.ll
@@ -0,0 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc --mtriple=i386-linux-gnu %s -o - -verify-machineinstrs \
+; RUN: | FileCheck %s
+; RUN: llc --mtriple=x86_64-linux-gnu %s -o - -verify-machineinstrs \
+; RUN: | FileCheck %s
+define void @x() fn_ret_thunk_extern {
+; CHECK-LABEL: x:
+; CHECK: # %bb.0:
+; CHECK-NEXT: jmp __x86_return_thunk
+ ret void
+}
diff --git a/llvm/test/CodeGen/X86/attr-function-return.mir b/llvm/test/CodeGen/X86/attr-function-return.mir
new file mode 100644
index 000000000000..91c03e862182
--- /dev/null
+++ b/llvm/test/CodeGen/X86/attr-function-return.mir
@@ -0,0 +1,62 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc --mtriple=x86_64-linux-gnu -run-pass=x86-return-thunks \
+# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+ ; ModuleID = 'y.ll'
+ source_filename = "y.ll"
+ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+ define void @x() #0 {
+ ret void
+ }
+
+ attributes #0 = { fn_ret_thunk_extern }
+
+...
+---
+name: x
+alignment: 16
+exposesReturnsTwice: false
+legalized: false
+regBankSelected: false
+selected: false
+failedISel: false
+tracksRegLiveness: true
+hasWinCFI: false
+failsVerification: false
+tracksDebugUserValues: true
+registers: []
+liveins: []
+frameInfo:
+ isFrameAddressTaken: false
+ isReturnAddressTaken: false
+ hasStackMap: false
+ hasPatchPoint: false
+ stackSize: 0
+ offsetAdjustment: 0
+ maxAlignment: 1
+ adjustsStack: false
+ hasCalls: false
+ stackProtector: ''
+ maxCallFrameSize: 0
+ cvBytesOfCalleeSavedRegisters: 0
+ hasOpaqueSPAdjustment: false
+ hasVAStart: false
+ hasMustTailInVarArgFunc: false
+ hasTailCall: false
+ localFrameSize: 0
+ savePoint: ''
+ restorePoint: ''
+fixedStack: []
+stack: []
+callSites: []
+debugValueSubstitutions: []
+constants: []
+machineFunctionInfo: {}
+body: |
+ bb.0 (%ir-block.0):
+ ; CHECK-LABEL: name: x
+ ; CHECK: TAILJMPd &__x86_return_thunk, implicit $esp, implicit $ssp
+ RET64
+
+...
diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll
index e83266f72e48..9c7f40696fbd 100644
--- a/llvm/test/CodeGen/X86/opt-pipeline.ll
+++ b/llvm/test/CodeGen/X86/opt-pipeline.ll
@@ -202,6 +202,7 @@
; CHECK-NEXT: Live DEBUG_VALUE analysis
; CHECK-NEXT: X86 Speculative Execution Side Effect Suppression
; CHECK-NEXT: X86 Indirect Thunks
+; CHECK-NEXT: X86 Return Thunks
; CHECK-NEXT: Check CFA info and insert CFI instructions if needed
; CHECK-NEXT: X86 Load Value Injection (LVI) Ret-Hardening
; CHECK-NEXT: Pseudo Probe Inserter
diff --git a/llvm/test/Transforms/Inline/attributes.ll b/llvm/test/Transforms/Inline/attributes.ll
index 07d1b36e485b..e668013926bf 100644
--- a/llvm/test/Transforms/Inline/attributes.ll
+++ b/llvm/test/Transforms/Inline/attributes.ll
@@ -600,6 +600,33 @@ define i32 @test_unsafe-fp-math3(i32 %i) "unsafe-fp-math"="true" {
; CHECK-NEXT: ret i32
}
+; Test that fn_ret_thunk_extern has no CompatRule; inlining is permitted.
+; Test that fn_ret_thunk_extern has no MergeRule; fn_ret_thunk_extern is not
+; propagated or dropped on the caller after inlining.
+define i32 @thunk_extern_callee() fn_ret_thunk_extern {
+; CHECK: @thunk_extern_callee() [[FNRETTHUNK_EXTERN:#[0-9]+]]
+ ret i32 42
+}
+
+define i32 @thunk_keep_caller() {
+; CHECK: @thunk_keep_caller() {
+; CHECK-NEXT: ret i32 42
+ %1 = call i32 @thunk_extern_callee()
+ ret i32 %1
+}
+
+define i32 @thunk_keep_callee() {
+; CHECK: @thunk_keep_callee() {
+ ret i32 42
+}
+
+define i32 @thunk_extern_caller() fn_ret_thunk_extern {
+; CHECK: @thunk_extern_caller() [[FNRETTHUNK_EXTERN]]
+; CHECK-NEXT: ret i32 42
+ %1 = call i32 @thunk_keep_callee()
+ ret i32 %1
+}
+
; CHECK: attributes [[SLH]] = { speculative_load_hardening }
; CHECK: attributes [[FPMAD_FALSE]] = { "less-precise-fpmad"="false" }
; CHECK: attributes [[FPMAD_TRUE]] = { "less-precise-fpmad"="true" }
@@ -614,3 +641,4 @@ define i32 @test_unsafe-fp-math3(i32 %i) "unsafe-fp-math"="true" {
; CHECK: attributes [[NO_SIGNED_ZEROS_FPMATH_TRUE]] = { "no-signed-zeros-fp-math"="true" }
; CHECK: attributes [[UNSAFE_FPMATH_FALSE]] = { "unsafe-fp-math"="false" }
; CHECK: attributes [[UNSAFE_FPMATH_TRUE]] = { "unsafe-fp-math"="true" }
+; CHECK: attributes [[FNRETTHUNK_EXTERN]] = { fn_ret_thunk_extern }
--
2.31.0