From a1df183218eb8bbcfe0bd21739351e976458f936 Mon Sep 17 00:00:00 2001
From: Marco Castelluccio <mcastelluccio@mozilla.com>
Date: Thu, 3 Oct 2019 03:52:15 +0000
Subject: [PATCH] Bug 1465709 - Hook rust OOM handler on rustc 1.28. r=froydnj

Bug 1458161 added a rust OOM handler based on an unstable API that was
removed in 1.27, replaced with something that didn't allow to get the
failed allocation size.

Latest 1.28 nightly (2018-06-13) has
https://github.com/rust-lang/rust/pull/50880,
https://github.com/rust-lang/rust/pull/51264 and
https://github.com/rust-lang/rust/pull/51241 merged, which allow to
hook the OOM handler and get the failed allocation size again.

Because this is still an unstable API, we explicitly depend on strict
versions of rustc. We also explicitly error out if automation builds
end up using a rustc version that doesn't allow us to get the allocation
size for rust OOM, because we don't want that to happen without knowing.

UltraBlame original commit: 5182bca90d0609f182d5a7b6b48ed2ffbbce32c2
---
 toolkit/crashreporter/nsExceptionHandler.cpp  |   9 ++
 toolkit/library/gtest/rust/Cargo.toml         |  11 ++
 toolkit/library/rust/Cargo.toml               |  11 ++
 toolkit/library/rust/gkrust-features.mozbuild |  85 ++++++++++++
 toolkit/library/rust/shared/Cargo.toml        |   4 +
 toolkit/library/rust/shared/build.rs          |   8 ++
 toolkit/library/rust/shared/lib.rs            | 127 ++++++++++++++++++
 7 files changed, 255 insertions(+)

diff --git a/toolkit/crashreporter/nsExceptionHandler.cpp b/toolkit/crashreporter/nsExceptionHandler.cpp
index 3ac36782380a2..8602f2d999cd3 100644
--- a/toolkit/crashreporter/nsExceptionHandler.cpp
+++ b/toolkit/crashreporter/nsExceptionHandler.cpp
@@ -765,6 +765,11 @@ install_rust_panic_hook
 (
 )
 ;
+void
+install_rust_oom_hook
+(
+)
+;
 bool
 get_rust_panic_reason
 (
@@ -7332,6 +7337,10 @@ install_rust_panic_hook
 (
 )
 ;
+install_rust_oom_hook
+(
+)
+;
 InitThreadAnnotation
 (
 )
diff --git a/toolkit/library/gtest/rust/Cargo.toml b/toolkit/library/gtest/rust/Cargo.toml
index 29497b759292e..5a2a5d26373d9 100644
--- a/toolkit/library/gtest/rust/Cargo.toml
+++ b/toolkit/library/gtest/rust/Cargo.toml
@@ -143,6 +143,17 @@ shared
 oom_with_global_alloc
 "
 ]
+oom_with_hook
+=
+[
+"
+gkrust
+-
+shared
+/
+oom_with_hook
+"
+]
 moz_memory
 =
 [
diff --git a/toolkit/library/rust/Cargo.toml b/toolkit/library/rust/Cargo.toml
index fa4e4ef0962f7..14dbfd001409a 100644
--- a/toolkit/library/rust/Cargo.toml
+++ b/toolkit/library/rust/Cargo.toml
@@ -141,6 +141,17 @@ shared
 oom_with_global_alloc
 "
 ]
+oom_with_hook
+=
+[
+"
+gkrust
+-
+shared
+/
+oom_with_hook
+"
+]
 moz_memory
 =
 [
diff --git a/toolkit/library/rust/gkrust-features.mozbuild b/toolkit/library/rust/gkrust-features.mozbuild
index 2f4df8f2babab..eeec42589b4fb 100644
--- a/toolkit/library/rust/gkrust-features.mozbuild
+++ b/toolkit/library/rust/gkrust-features.mozbuild
@@ -310,3 +310,88 @@ gkrust_features
 oom_with_global_alloc
 '
 ]
+elif
+CONFIG
+[
+'
+RUSTC_VERSION
+'
+]
+>
+=
+"
+1
+.
+28
+"
+and
+CONFIG
+[
+'
+RUSTC_VERSION
+'
+]
+<
+"
+1
+.
+29
+"
+:
+gkrust_features
++
+=
+[
+'
+oom_with_hook
+'
+]
+elif
+not
+CONFIG
+[
+'
+MOZ_AUTOMATION
+'
+]
+:
+#
+We
+don
+'
+t
+want
+builds
+on
+automation
+to
+unwillingly
+stop
+annotating
+OOM
+#
+crash
+reports
+from
+rust
+.
+error
+(
+'
+Builds
+on
+automation
+must
+use
+a
+version
+of
+rust
+that
+supports
+OOM
+'
+'
+hooking
+'
+)
diff --git a/toolkit/library/rust/shared/Cargo.toml b/toolkit/library/rust/shared/Cargo.toml
index c844ec6ad96c4..c53979dad3522 100644
--- a/toolkit/library/rust/shared/Cargo.toml
+++ b/toolkit/library/rust/shared/Cargo.toml
@@ -679,6 +679,10 @@ oom_with_global_alloc
 =
 [
 ]
+oom_with_hook
+=
+[
+]
 moz_memory
 =
 [
diff --git a/toolkit/library/rust/shared/build.rs b/toolkit/library/rust/shared/build.rs
index 19dfbe51648d1..cd426f97cf9c4 100644
--- a/toolkit/library/rust/shared/build.rs
+++ b/toolkit/library/rust/shared/build.rs
@@ -7,11 +7,19 @@ main
 [
 cfg
 (
+any
+(
 feature
 =
 "
 oom_with_global_alloc
 "
+feature
+=
+"
+oom_with_hook
+"
+)
 )
 ]
 println
diff --git a/toolkit/library/rust/shared/lib.rs b/toolkit/library/rust/shared/lib.rs
index a522bd8e871f5..d048624aece94 100644
--- a/toolkit/library/rust/shared/lib.rs
+++ b/toolkit/library/rust/shared/lib.rs
@@ -18,6 +18,22 @@ allocator_api
 )
 ]
 #
+!
+[
+cfg_attr
+(
+feature
+=
+"
+oom_with_hook
+"
+feature
+(
+oom_hook
+)
+)
+]
+#
 [
 cfg
 (
@@ -1153,3 +1169,114 @@ global_alloc
 :
 GeckoHeap
 ;
+#
+[
+cfg
+(
+feature
+=
+"
+oom_with_hook
+"
+)
+]
+mod
+oom_hook
+{
+use
+std
+:
+:
+alloc
+:
+:
+{
+Layout
+set_oom_hook
+}
+;
+extern
+"
+C
+"
+{
+fn
+GeckoHandleOOM
+(
+size
+:
+usize
+)
+-
+>
+!
+;
+}
+pub
+fn
+hook
+(
+layout
+:
+Layout
+)
+{
+unsafe
+{
+GeckoHandleOOM
+(
+layout
+.
+size
+(
+)
+)
+;
+}
+}
+pub
+fn
+install
+(
+)
+{
+set_oom_hook
+(
+hook
+)
+;
+}
+}
+#
+[
+no_mangle
+]
+pub
+extern
+"
+C
+"
+fn
+install_rust_oom_hook
+(
+)
+{
+#
+[
+cfg
+(
+feature
+=
+"
+oom_with_hook
+"
+)
+]
+oom_hook
+:
+:
+install
+(
+)
+;
+}