Deadlock on armv7-unknown-linux-gnueabihf

[pgerber]

On armv7-unknown-linux-gnueabihf the following code leads to a deadlock:

extern crate hyper_openssl;

use hyper_openssl::OpensslClient;

fn main() {
    OpensslClient::new().unwrap();
}

Backtrace:

(gdb) bt
#0  0x76dae780 in __lll_lock_wait (futex=futex@entry=0x76817150, private=<optimized out>) at ../ports/sysdeps/unix/sysv/linux/arm/nptl/lowlevellock.c:46
#1  0x76da91a4 in __GI___pthread_mutex_lock (mutex=0x76817150) at pthread_mutex_lock.c:79
#2  0x54ab5260 in std::sys::imp::mutex::{{impl}}::lock (self=0x76817150) at /checkout/src/libstd/sys/unix/mutex.rs:67
#3  0x54ab4dac in std::sys_common::mutex::{{impl}}::lock (self=0x76817150) at /checkout/src/libstd/sys_common/mutex.rs:40
#4  0x54ab5bd4 in std::sync::mutex::{{impl}}::lock<()> (self=0x7682c260) at /checkout/src/libstd/sync/mutex.rs:221
#5  0x54ab8c9c in openssl_sys::ossl10x::locking_function (mode=9, n=12, _file=0x76fa065c "ssl_cert.c", _line=146) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-sys-0.9.8/src/ossl10x.rs:720
#6  0x76f93ab8 in SSL_get_ex_data_X509_STORE_CTX_idx () from /usr/lib/arm-linux-gnueabihf/libssl.so.1.0.0
#7  0x76f91380 in SSL_CTX_new () from /usr/lib/arm-linux-gnueabihf/libssl.so.1.0.0
#8  0x54ab3188 in openssl::ssl::{{impl}}::new (method=...) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.8/src/ssl/mod.rs:598
#9  0x54ab2ac8 in openssl::ssl::connector::ctx (method=...) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.8/src/ssl/connector.rs:23
#10 0x54ab2ddc in openssl::ssl::connector::{{impl}}::new (method=...) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.8/src/ssl/connector.rs:52
#11 0x54aae980 in hyper_openssl::{{impl}}::new () at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/hyper-openssl-0.2.3/src/lib.rs:72
#12 0x54aae87c in ossl::main () at /home/pg/rust/ossl/src/main.rs:7
#13 0x54ac28f0 in std::panicking::try::do_call<fn(),()> () at /checkout/src/libstd/panicking.rs:454
#14 0x54ac75b0 in panic_unwind::__rust_maybe_catch_panic () at /checkout/src/libpanic_unwind/lib.rs:98
#15 0x54ac3178 in try<(),fn()> () at /checkout/src/libstd/panicking.rs:433
#16 catch_unwind<fn(),()> () at /checkout/src/libstd/panic.rs:361
#17 std::rt::lang_start () at /checkout/src/libstd/rt.rs:57
#18 0x54aae8d0 in main ()

OS:

$ lsb_release -a
No LSB modules are available.
Distributor ID: Raspbian
Description:    Raspbian GNU/Linux 8.0 (jessie)
Release:        8.0
Codename:       jessie

[sfackler]

That's pretty weird. If you stick a breakpoint on `locking_function, what do the call patterns look like?

[pgerber]

I agree, it's weird.

Tried different rustc versions, only the beta and nightly channel appear to be affected.

Call Pattern:

(gdb) break openssl_sys::ossl10x::locking_function
Breakpoint 1 at 0xec24: file /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-sys-0.9.9/src/ossl10x.rs, line 715.

(gdb) r
Starting program: /home/pg/rust/ossl-nightly/target/debug/ossl
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/arm-linux-gnueabihf/libthread_db.so.1".
Cannot access memory at address 0x0

Program received signal SIGILL, Illegal instruction.
0x76e36de8 in ?? () from /usr/lib/arm-linux-gnueabihf/libcrypto.so.1.0.0
(gdb) c
Continuing.

Breakpoint 1, openssl_sys::ossl10x::locking_function (mode=1996034944, n=1996183588, _file=0x2 <error: Cannot access memory at address 0x2>, _line=2130701928) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-sys-0.9.9/src/ossl10x.rs:715
715     unsafe extern fn locking_function(mode: c_int, n: c_int, _file: *const c_char,
(gdb) bt
#0  openssl_sys::ossl10x::locking_function (mode=1996034944, n=1996183588, _file=0x2 <error: Cannot access memory at address 0x2>, _line=2130701928) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-sys-0.9.9/src/ossl10x.rs:715
#1  0x76f93a60 in SSL_get_ex_data_X509_STORE_CTX_idx () from /usr/lib/arm-linux-gnueabihf/libssl.so.1.0.0
#2  0x76f91380 in SSL_CTX_new () from /usr/lib/arm-linux-gnueabihf/libssl.so.1.0.0
#3  0x54ab3188 in openssl::ssl::{{impl}}::new (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/mod.rs:598
#4  0x54ab2ac8 in openssl::ssl::connector::ctx (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/connector.rs:23
#5  0x54ab2ddc in openssl::ssl::connector::{{impl}}::new (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/connector.rs:52
#6  0x54aae980 in hyper_openssl::{{impl}}::new () at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/hyper-openssl-0.2.3/src/lib.rs:72
#7  0x54aae87c in ossl::main () at /home/pg/rust/ossl-nightly/src/main.rs:6
#8  0x54ac28f0 in std::panicking::try::do_call<fn(),()> () at /checkout/src/libstd/panicking.rs:454
#9  0x54ac75b0 in panic_unwind::__rust_maybe_catch_panic () at /checkout/src/libpanic_unwind/lib.rs:98
#10 0x54ac3178 in try<(),fn()> () at /checkout/src/libstd/panicking.rs:433
#11 catch_unwind<fn(),()> () at /checkout/src/libstd/panic.rs:361
#12 std::rt::lang_start () at /checkout/src/libstd/rt.rs:57
#13 0x54aae8d0 in main ()
(gdb) c
Continuing.

Breakpoint 1, openssl_sys::ossl10x::locking_function (mode=1996034944, n=1996183588, _file=0x2 <error: Cannot access memory at address 0x2>, _line=2130701928) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-sys-0.9.9/src/ossl10x.rs:715
715     unsafe extern fn locking_function(mode: c_int, n: c_int, _file: *const c_char,
(gdb) bt
#0  openssl_sys::ossl10x::locking_function (mode=1996034944, n=1996183588, _file=0x2 <error: Cannot access memory at address 0x2>, _line=2130701928) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-sys-0.9.9/src/ossl10x.rs:715
#1  0x76f93aa4 in SSL_get_ex_data_X509_STORE_CTX_idx () from /usr/lib/arm-linux-gnueabihf/libssl.so.1.0.0
#2  0x76f91380 in SSL_CTX_new () from /usr/lib/arm-linux-gnueabihf/libssl.so.1.0.0
#3  0x54ab3188 in openssl::ssl::{{impl}}::new (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/mod.rs:598
#4  0x54ab2ac8 in openssl::ssl::connector::ctx (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/connector.rs:23
#5  0x54ab2ddc in openssl::ssl::connector::{{impl}}::new (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/connector.rs:52
#6  0x54aae980 in hyper_openssl::{{impl}}::new () at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/hyper-openssl-0.2.3/src/lib.rs:72
#7  0x54aae87c in ossl::main () at /home/pg/rust/ossl-nightly/src/main.rs:6
#8  0x54ac28f0 in std::panicking::try::do_call<fn(),()> () at /checkout/src/libstd/panicking.rs:454
#9  0x54ac75b0 in panic_unwind::__rust_maybe_catch_panic () at /checkout/src/libpanic_unwind/lib.rs:98
#10 0x54ac3178 in try<(),fn()> () at /checkout/src/libstd/panicking.rs:433
#11 catch_unwind<fn(),()> () at /checkout/src/libstd/panic.rs:361
#12 std::rt::lang_start () at /checkout/src/libstd/rt.rs:57
#13 0x54aae8d0 in main ()
(gdb) c
Continuing.

Breakpoint 1, openssl_sys::ossl10x::locking_function (mode=1996034944, n=1996183588, _file=0x2 <error: Cannot access memory at address 0x2>, _line=2130701928) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-sys-0.9.9/src/ossl10x.rs:715
715     unsafe extern fn locking_function(mode: c_int, n: c_int, _file: *const c_char,
(gdb) bt
#0  openssl_sys::ossl10x::locking_function (mode=1996034944, n=1996183588, _file=0x2 <error: Cannot access memory at address 0x2>, _line=2130701928) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-sys-0.9.9/src/ossl10x.rs:715
#1  0x76f93ab8 in SSL_get_ex_data_X509_STORE_CTX_idx () from /usr/lib/arm-linux-gnueabihf/libssl.so.1.0.0
#2  0x76f91380 in SSL_CTX_new () from /usr/lib/arm-linux-gnueabihf/libssl.so.1.0.0
#3  0x54ab3188 in openssl::ssl::{{impl}}::new (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/mod.rs:598
#4  0x54ab2ac8 in openssl::ssl::connector::ctx (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/connector.rs:23
#5  0x54ab2ddc in openssl::ssl::connector::{{impl}}::new (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/connector.rs:52
#6  0x54aae980 in hyper_openssl::{{impl}}::new () at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/hyper-openssl-0.2.3/src/lib.rs:72
#7  0x54aae87c in ossl::main () at /home/pg/rust/ossl-nightly/src/main.rs:6
#8  0x54ac28f0 in std::panicking::try::do_call<fn(),()> () at /checkout/src/libstd/panicking.rs:454
#9  0x54ac75b0 in panic_unwind::__rust_maybe_catch_panic () at /checkout/src/libpanic_unwind/lib.rs:98
#10 0x54ac3178 in try<(),fn()> () at /checkout/src/libstd/panicking.rs:433
#11 catch_unwind<fn(),()> () at /checkout/src/libstd/panic.rs:361
#12 std::rt::lang_start () at /checkout/src/libstd/rt.rs:57
#13 0x54aae8d0 in main ()
(gdb) c
Continuing.
^C
Program received signal SIGINT, Interrupt.
0x76dae780 in __lll_lock_wait (futex=futex@entry=0x76817150, private=<optimized out>) at ../ports/sysdeps/unix/sysv/linux/arm/nptl/lowlevellock.c:46
46      ../ports/sysdeps/unix/sysv/linux/arm/nptl/lowlevellock.c: No such file or directory.
(gdb) bt
#0  0x76dae780 in __lll_lock_wait (futex=futex@entry=0x76817150, private=<optimized out>) at ../ports/sysdeps/unix/sysv/linux/arm/nptl/lowlevellock.c:46
#1  0x76da91a4 in __GI___pthread_mutex_lock (mutex=0x76817150) at pthread_mutex_lock.c:79
#2  0x54ab5260 in std::sys::imp::mutex::{{impl}}::lock (self=0x76817150) at /checkout/src/libstd/sys/unix/mutex.rs:67
#3  0x54ab4dac in std::sys_common::mutex::{{impl}}::lock (self=0x76817150) at /checkout/src/libstd/sys_common/mutex.rs:40
#4  0x54ab5bd4 in std::sync::mutex::{{impl}}::lock<()> (self=0x7682c260) at /checkout/src/libstd/sync/mutex.rs:221
#5  0x54ab8c9c in openssl_sys::ossl10x::locking_function (mode=9, n=12, _file=0x76fa065c "ssl_cert.c", _line=146) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-sys-0.9.9/src/ossl10x.rs:720
#6  0x76f93ab8 in SSL_get_ex_data_X509_STORE_CTX_idx () from /usr/lib/arm-linux-gnueabihf/libssl.so.1.0.0
#7  0x76f91380 in SSL_CTX_new () from /usr/lib/arm-linux-gnueabihf/libssl.so.1.0.0
#8  0x54ab3188 in openssl::ssl::{{impl}}::new (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/mod.rs:598
#9  0x54ab2ac8 in openssl::ssl::connector::ctx (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/connector.rs:23
#10 0x54ab2ddc in openssl::ssl::connector::{{impl}}::new (method=SslMethod = {...}) at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/openssl-0.9.9/src/ssl/connector.rs:52
#11 0x54aae980 in hyper_openssl::{{impl}}::new () at /home/pg/.cargo/registry/src/github.com-1ecc6299db9ec823/hyper-openssl-0.2.3/src/lib.rs:72
#12 0x54aae87c in ossl::main () at /home/pg/rust/ossl-nightly/src/main.rs:6
#13 0x54ac28f0 in std::panicking::try::do_call<fn(),()> () at /checkout/src/libstd/panicking.rs:454
#14 0x54ac75b0 in panic_unwind::__rust_maybe_catch_panic () at /checkout/src/libpanic_unwind/lib.rs:98
#15 0x54ac3178 in try<(),fn()> () at /checkout/src/libstd/panicking.rs:433
#16 catch_unwind<fn(),()> () at /checkout/src/libstd/panic.rs:361
#17 std::rt::lang_start () at /checkout/src/libstd/rt.rs:57
#18 0x54aae8d0 in main ()

[sfackler]

Huh, the arguments to locking_function on all the calls but the last one look very wrong. If you step into the function a couple of times do they clear up?

The fact that it's only on nightly/beta is concerning. @alexcrichton do you know of anything off the top of your head that could be going on here?

[sfackler]

Try adding a simple println! of all of the arguments to locking_function.

[pgerber]

Added a println!:

mode: 5, n: 12, _file: 0x76f0765c ("ssl_cert.c"), _line: 142
mode: 6, n: 12, _file: 0x76f0765c ("ssl_cert.c"), _line: 145
mode: 9, n: 12, _file: 0x76f0765c ("ssl_cert.c"), _line: 146

For some reason one line is missing (the last call). First assumed a stdout().flush() was missing but that didn't help.

[sfackler]

I don't think anything's missing - the GDB output above seems to be showing three calls as well. The weird thing is that it's a lock, unlock, and lock, so I'm not sure why that would be problematic. It might be worth seeing if the GUARDS array looks as expected?

[pgerber]

You're right, looks like I lose the ability to count to three past midnight.

Added some more println!s:

unsafe extern fn locking_function(mode: c_int, n: c_int, _file: *const c_char,
                               _line: c_int) {
    let file_name = CStr::from_ptr(_file);
    println!("mode: {:?}, n: {:?}, _file: {:?} ({:?}), _line: {:?}", mode, n, _file, file_name, _line);
    println!("MUTEXES: {:?}", (*MUTEXES)[n as usize]);
    println!("GUARDS: {:?}", (*GUARDS)[n as usize]);
    println!("::CRYPTO_LOCK: {:?}", ::CRYPTO_LOCK);
    println!();

    …

and here's the output:

mode: 5, n: 12, _file: 0x76f9965c ("ssl_cert.c"), _line: 142
MUTEXES: Mutex { data: () }
GUARDS: None
::CRYPTO_LOCK: 1

mode: 6, n: 12, _file: 0x76f9965c ("ssl_cert.c"), _line: 145
MUTEXES: Mutex { <locked> }
GUARDS: Some(MutexGuard { lock: Mutex { <locked> } })
::CRYPTO_LOCK: 1

mode: 9, n: 12, _file: 0x76f9965c ("ssl_cert.c"), _line: 146
MUTEXES: Mutex { <locked> }
GUARDS: None
::CRYPTO_LOCK: 1

[alexcrichton]

@sfackler the biggest change is how we build binaries, moving from buildbot to Travis. Maybe some build system changes induced a toolchain difference for arm causing problems? Other than that nothing comes to mind.

[sfackler]

The output up there is concerning - the mutex is registered as locked even though the guard is no longer there. Could you try running this little test to see if mutexes are just broken for you?

use std::sync::Mutex;

fn main() {
    let lock = Mutex::new(());
    let guard = lock.lock().unwrap();
    drop(guard);
    lock.lock().unwrap();
}

[pgerber]

@sfackler Your test program works as expected, no deadlock.

[sfackler]

:/

[pgerber]

I'll go through a few different versions of rustc and rust-openssl in the hope that I can figure out when that issue was introduced.

[sfackler]

Cool. I have always felt a bit uncomfortable with this bit of code - I'm guessing either the code is incorrect and some changes in rustc or llvm have started taking advantage of things that break it, or there's some compiler bug and the code is correct.

I don't have access to any ARM hardware unfortunately, so the debugging will probably have to happen on your end unfortunately :(

One thing that would be valuable is to minimize down to some reasonably small bit of code that exhibits the issue. My guess is that it'll be this dual-static-vector setup, but it may also have other conditions like being called through the C ABI?

[pgerber]

As suggested by you I minimized the code.

This here fails:

let mutex = Mutex::new(());
let mut guard = None;
guard = Some(mutex.lock().unwrap());
drop(guard.take());
mutex.try_lock().unwrap(); // panics "WouldBlock"

This here works:

let mutex = Mutex::new(());
let mut guard = None;
guard = Some(mutex.lock().unwrap());
println!("guard {:?}", guard);
drop(guard.take().unwrap());   // unwrap() added here
mutex.try_lock().unwrap();     // locks as expected

As far as I can tell the MutexGuard owned by Some is not properly dropped. I'll try to figure out what compiler version introduced this issue next week.

In the meantime, I'll make a pull request adding an unwrap. Even if it didn't "fix" the deadlock, I think it would be a good idea to have it in there so accidenly double-unlocking panics.

[sfackler]

Cool, that looks like something we can send over to rust-lang/rust.

Unwrapping the guard option seems like a good idea to me though!