Panic Cause: null pointer dereference in c4socket_gotHTTPResponse

[CedricCouton]

Hi,

During our test on android we have sometimes this issue on android :

Build fingerprint: 'samsung/a40eea/a40:10/QP1A.190711.020/A405FNXXS3BTI3:user/release-keys'
Revision: '4'
ABI: 'arm64'
Timestamp: 2020-12-16 12:13:39+0100
pid: 19962, tid: 20509, name: Executor  >>> fr.test.dev <<<
uid: 10270
signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0x48
Cause: null pointer dereference
    x0  0000007b172a98e0  x1  0000000000000065  x2  0000007a82bdc3c8  x3  0000007aaffb99bc
    x4  0000000000000004  x5  0000000000000000  x6  6c6f636f746f7270  x7  335f50494c42114f
    x8  0000000000000000  x9  0000000000000000  x10 0000000000000002  x11 0000007b173110d8
    x12 0000000000000000  x13 fffffffffc000000  x14 0000000000000040  x15 0000008000000000
    x16 0000007ab0271270  x17 0000007ab018b6bc  x18 00000071024b6000  x19 0000007b1d4ef618
    x20 0000000000000065  x21 0000007a82bdc3c8  x22 0000007a82be3020  x23 0000000000000000
    x24 0000007a86bda930  x25 0000007a86bda930  x26 0000007a86bdbc10  x27 0000007a86bdae30
    x28 000000007b1d4c02  x29 0000007a82bdc5a0
    sp  0000007a82bdc3b0  lr  0000007aaff32934  pc  0000007aaffb99c0

backtrace:
      #00 pc 000000000016d9c0  /data/app/fr.test.dev-4OijczSr3wtee3d-UfH9BQ==/lib/arm64/libLiteCore.so (BuildId: 89adae4eb5fd805cff80e4d9992c83834c63e635)
      #01 pc 00000000000e6930  /data/app/fr.test.dev-4OijczSr3wtee3d-UfH9BQ==/lib/arm64/libLiteCore.so (c4socket_gotHTTPResponse+100) (BuildId: 89adae4eb5fd805cff80e4d9992c83834c63e635)

When I look the code of c4socket_gotHTTPResponse (c4Socket.cc) I can see a try catch but I don't know if it catch a NPE ?

void c4socket_gotHTTPResponse(C4Socket *socket, int status, C4Slice responseHeadersFleece) C4API {
    try {
        Headers headers(responseHeadersFleece);
        internal(socket)->gotHTTPResponse(status, headers);
    } catchForSocket(socket)
}

In cbl-rust this function is used in repl_transport.rs like this :

c4socket_gotHTTPResponse(
        sock_id as *mut _,
        http_resp.status().as_u16() as c_int,
        headers.as_flslice(),
    );

My first question is, is it possible that sock_id can be null ?
Second question, do you know if checking that sock_id is not a null pointer could be a good idea ?

[Dushistov]

My first question is, is it possible that sock_id can be null ?

c4socket_gotHTTPResponse is used only in open_connection, which is called from ws_open,
where there is assert!(!c4sock.is_null());.

So even if there are some bugs in C++ or in Rust part the sock_id can be (in theory) point to already freed memory or something like this, but it should not be null pointer.

[Dushistov]

It should be SIGABRT (signal 6) because of assert if c4socket is null pointer

[CedricCouton]

Have you an idea where the nullpointer dereference is thrown?

[CedricCouton]

00000000000e68cc c4socket_gotHTTPResponse:
; void c4socket_gotHTTPResponse(C4Socket *socket, int status, C4Slice responseHeadersFleece) C4API {
   e68cc: fc 0f 1c f8                   str     x28, [sp, #-64]!
   e68d0: f6 57 01 a9                   stp     x22, x21, [sp, #16]
   e68d4: f4 4f 02 a9                   stp     x20, x19, [sp, #32]
   e68d8: fd 7b 03 a9                   stp     x29, x30, [sp, #48]
   e68dc: fd c3 00 91                   add     x29, sp, #48
   e68e0: ff 03 07 d1                   sub     sp, sp, #448
   e68e4: 56 d0 3b d5                   mrs     x22, TPIDR_EL0
   e68e8: c8 16 40 f9                   ldr     x8, [x22, #40]
   e68ec: f4 03 01 2a                   mov     w20, w1
   e68f0: f3 03 00 aa                   mov     x19, x0
   e68f4: a8 83 1c f8                   stur    x8, [x29, #-56]
;         explicit Headers(slice encoded)                 :Headers(alloc_slice(encoded)) { }
   e68f8: e0 23 00 91                   add     x0, sp, #8
   e68fc: e1 03 02 aa                   mov     x1, x2
   e6900: e2 03 03 aa                   mov     x2, x3
   e6904: c9 6c 09 94                   bl      #2470692 <_ZN6fleece11alloc_sliceC2ENS_10pure_sliceE>
   e6908: e0 63 00 91                   add     x0, sp, #24
   e690c: e1 23 00 91                   add     x1, sp, #8
   e6910: 14 3e 02 94                   bl      #587856 <_ZN8litecore9websocket7HeadersC2EN6fleece11alloc_sliceE>
;         ~alloc_slice() noexcept                             {if (buf) release();}
   e6914: e0 07 40 f9                   ldr     x0, [sp, #8]
   e6918: 40 00 00 b4                   cbz     x0, #8 <c4socket_gotHTTPResponse+0x54>
;     _FLBuf_Release(s.buf);
   e691c: d5 a0 fd 97                   bl      #-621740 <$x+0xc20>
; static C4SocketImpl* internal(C4Socket *s)  {return (C4SocketImpl*)s;}
   e6920: 60 62 05 d1                   sub     x0, x19, #344
;         internal(socket)->gotHTTPResponse(status, headers);
   e6924: e2 63 00 91                   add     x2, sp, #24
   e6928: e1 03 14 2a                   mov     w1, w20
   e692c: f5 63 00 91                   add     x21, sp, #24
   e6930: 36 44 02 94                   bl      #594136 <_ZN8litecore9websocket13WebSocketImpl15gotHTTPResponseEiRKNS0_7HeadersE>
;     class Headers {
   e6934: a0 a2 00 91                   add     x0, x21, #40
   e6938: fe b6 01 94                   bl      #449528 <_ZN6fleece6WriterD2Ev>
;         ~alloc_slice() noexcept                             {if (buf) release();}
   e693c: e0 1b 40 f9                   ldr     x0, [sp, #48]
   e6940: 40 00 00 b4                   cbz     x0, #8 <c4socket_gotHTTPResponse+0x7c>
;     _FLBuf_Release(s.buf);
   e6944: cb a0 fd 97                   bl      #-621780 <$x+0xc20>
;         {return static_cast<__node_pointer>(__end_node()->__left_);}
   e6948: e1 13 40 f9                   ldr     x1, [sp, #32]
;   destroy(__root());
   e694c: e0 63 00 91                   add     x0, sp, #24
   e6950: 88 01 00 94                   bl      #1568 <_ZNSt6__ndk16__treeINS_12__value_typeIN6fleece5sliceES3_EENS_19__map_value_compareIS3_S4_N8litecore9websocket7Headers9HeaderCmpELb1EEENS_9allocatorIS4_EEE7destroyEPNS_11__tree_nodeIS4_PvEE>
   e6954: c8 16 40 f9                   ldr     x8, [x22, #40]
   e6958: a9 83 5c f8                   ldur    x9, [x29, #-56]
   e695c: 1f 01 09 eb                   cmp     x8, x9
   e6960: e1 00 00 54                   b.ne    #28 <c4socket_gotHTTPResponse+0xb0>
; }

[CedricCouton]

Can you remind me why did you implement your own transport layer and not use the BuiltInWebSocket ? To support the wss protocol ? thanks

[CedricCouton]

If I do that :

aarch64-linux-android-addr2line  -C -f -e arm64-v8a/libLiteCore.so  000000000016d9c0
litecore::blip::Connection::delegate() const
couchbase-lite-core-sys/couchbase-lite-core/Networking/BLIP/BLIPConnection.hh:63

that say it's in the delegate function ?

[Dushistov]

Can you remind me why did you implement your own transport layer and not use the BuiltInWebSocket ? To support the wss protocol ?

Because the crate is designed to use in Rust application. The tokio by default run N (where N is number of logical cores) threads with epoll event loops,
the couchbase-lite-core (BuiltInWebSocket) also create N threads with epoll event loops.

One "network runtime" is more simple to control.

that say it's in the delegate function ?

Not sure about this, shared library can be mapped to any address,
so to find out which function and instruction at address 000000000016d9c0 you need
to know base address of shared library. In case of aarch64-linux-android-addr2line there is assumption
that base addresss of shared library is zero, but Android should have enabled "Address space layout randomization" (ASLR).

If you can provide something to reproduce problem on my side I will help you to fix.
For example if you run https://github.com/Dushistov/couchbase-lite-rust/tree/master/chat-demo on the two phones,
can you reproduce problem?

On android you can run binary via:

adb push "$EXE" "/data/local/tmp/$EXE_NAME"
adb shell "chmod 755 /data/local/tmp/$EXE_NAME"
adb shell "cd /data/local/tmp/ && RUST_LOG=debug /data/local/tmp/$EXE_NAME"

[CedricCouton]

The problem of the problem is that occurred rarely. The new news is that occurred also on desktop macos (10.15) and I think at the same place. Our app is running with electronjs where the native lib is called via ffi-napi. Rarely the app cashed, the crashdump indicate :

Operating system: Mac OS X
                  10.15.7 19H2
CPU: amd64
     family 6 model 142 stepping 10
     8 CPUs

GPU: UNKNOWN

Crash reason:  EXC_BAD_ACCESS / KERN_INVALID_ADDRESS
Crash address: 0x800000010
Process uptime: 6609 seconds

Thread 39 (crashed)
 0  libLiteCore.dylib + 0x3540f0
    rax = 0x00007f86505062e0   rdx = 0x0000700015370b08
    rcx = 0x0000000800000000   rbx = 0x00007f86485a6424
    rsi = 0x0000000000000065   rdi = 0x00007f86505062e0
    rbp = 0x0000700015370a90   rsp = 0x0000700015370a70
     r8 = 0x0000000000000000    r9 = 0x0000000000000d00
    r10 = 0x00000000fff00000   r11 = 0xfffffffffff6b700
    r12 = 0x0000700015375438   r13 = 0x00007f864b03a440
    r14 = 0x000000012266a210   r15 = 0x00007f864b039f30
    rip = 0x000000011f9180f0
    Found by: given as instruction pointer in context
 1  libLiteCore.dylib + 0x18e3d7
    rbp = 0x0000700015370cb0   rsp = 0x0000700015370aa0
    rip = 0x000000011f7523d7

When i used the atos command line to know where the crash had take place, I found that :

➜  .doctolib-billing atos -o $HOME/Downloads/libLiteCore\ \(1\).dylib 0x3540f0 -arch x86_64
atos[23356]: respawning is disabled (because DYLD_ROOT_PATH or DT_NO_RESPAWN is set), but the analysis process does not match the SDK variant of the target process 0.
Analysis of malloc zones may fail.
litecore::websocket::WebSocketImpl::gotHTTPResponse(int, litecore::websocket::Headers const&) (in libLiteCore (1).dylib) (WebSocketImpl.cc:103)

and

➜  .doctolib-billing atos -o $HOME/Downloads/libLiteCore\ \(1\).dylib 0x18e3d7 -arch x86_64
atos[23801]: respawning is disabled (because DYLD_ROOT_PATH or DT_NO_RESPAWN is set), but the analysis process does not match the SDK variant of the target process 0.
Analysis of malloc zones may fail.
c4socket_gotHTTPResponse (in libLiteCore (1).dylib) (c4Socket.cc:207)

Curiously it seem to be the same place as on mobile.

I tried to reproduce it with a stress test which loop to start replication and stop replication but I never had the crash. I will continue to try... If you have some ideas I'm all hears

[CedricCouton]

Today I have the save problem in dev env. Sometimes my app crash but I never analyzed the dump. Today I did it and :

Thread 20 Crashed:
0   daemon                              0x000000010909a0cc litecore::blip::Connection::delegate() const + 12 (BLIPConnection.hh:63)
1   daemon                              0x000000010909a09c litecore::blip::Connection::gotHTTPResponse(int, litecore::websocket::Headers const&) + 28 (BLIPConnection.cc:688)
2   daemon                              0x000000010909ba84 litecore::blip::BLIPIO::onWebSocketGotHTTPResponse(int, litecore::websocket::Headers const&) + 52 (BLIPConnection.cc:184)
3   daemon                              0x00000001090bae03 litecore::websocket::WebSocketImpl::gotHTTPResponse(int, litecore::websocket::Headers const&) + 131 (WebSocketImpl.cc:104)
4   daemon                              0x0000000108f47c27 c4socket_gotHTTPResponse + 151 (c4Socket.cc:207)
5   daemon                              0x0000000108c8faed couchbase_lite::repl_transport::open_connection::_$u7b$$u7b$closure$u7d$$u7d$::h0deaae0780c40a23 + 3853 (repl_transport.rs:403)
6   daemon                              0x0000000108ce2850 _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::he647e09b9e051382 + 80 (mod.rs:80)
7   daemon                              0x0000000108c89136 couchbase_lite::repl_transport::ws_open::_$u7b$$u7b$closure$u7d$$u7d$::hf2ca7db9f9f3ed9b + 326 (repl_transport.rs:139)
8   daemon                              0x0000000108ce23e0 _$LT$core..future..from_generator..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::ha7eecb9b83014447 + 80 (mod.rs:80)
9   daemon                              0x0000000108ce82fa tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::_$u7b$$u7b$closure$u7d$$u7d$::h400fbc55ddd0f710 + 298 (core.rs:173)
10  daemon                              0x0000000108caea30 tokio::loom::std::unsafe_cell::UnsafeCell$LT$T$GT$::with_mut::hfacaf79820f959c7 + 64 (unsafe_cell.rs:14)
11  daemon                              0x0000000108ce7e71 tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::h6656180304bfb690 + 49 (core.rs:158)
12  daemon                              0x0000000108ccbdc7 tokio::runtime::task::harness::Harness$LT$T$C$S$GT$::poll::_$u7b$$u7b$closure$u7d$$u7d$::h8d6465517fdb79a0 + 263 (harness.rs:107)
13  daemon                              0x0000000108cbd0f0 core::ops::function::FnOnce::call_once::he8f52ab0fb4e6e65 + 32 (function.rs:227)
14  daemon                              0x0000000108cea32c _$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h0e7ae264d1a7c57f + 28
15  daemon                              0x0000000108cb6586 std::panicking::try::do_call::h25c964e2e86f1748 + 86 (panicking.rs:381)
16  daemon                              0x0000000108cbbaed __rust_try + 29
17  daemon                              0x0000000108cb5a47 std::panicking::try::hb5d9677f0d467d7e + 135 (panicking.rs:345)
18  daemon                              0x0000000108cee21c std::panic::catch_unwind::h39665fd2b13b913c + 28
19  daemon                              0x0000000108cca41c tokio::runtime::task::harness::Harness$LT$T$C$S$GT$::poll::h0208f423ab19df25 + 380 (harness.rs:117)
20  daemon                              0x0000000108ce0c82 tokio::runtime::task::raw::poll::he6f99e462c94921b + 34 (raw.rs:105)
21  daemon                              0x00000001091496ce tokio::runtime::task::raw::RawTask::poll::h2f2df4023a2996d7 + 46 (raw.rs:67)
22  daemon                              0x0000000108c6a83d tokio::runtime::task::Notified$LT$S$GT$::run::haaaae37c2cbef242 + 29 (mod.rs:169)
23  daemon                              0x0000000108c51831 tokio::runtime::basic_scheduler::BasicScheduler$LT$P$GT$::block_on::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h09283c5f6f0b5420 + 17 (basic_scheduler.rs:155)
24  daemon                              0x0000000108c7d062 tokio::coop::with_budget::_$u7b$$u7b$closure$u7d$$u7d$::h666a4c4072694ab0 + 146 (coop.rs:127)
25  daemon                              0x0000000108c6ede6 std::thread::local::LocalKey$LT$T$GT$::try_with::h0c6f4d86cd2ea789 + 166 (local.rs:272)
26  daemon                              0x0000000108c6eb34 std::thread::local::LocalKey$LT$T$GT$::with::h04222338e77ab13a + 36 (local.rs:248)
27  daemon                              0x0000000108c5158f tokio::runtime::basic_scheduler::BasicScheduler$LT$P$GT$::block_on::_$u7b$$u7b$closure$u7d$$u7d$::hbec478b19dda3594 + 1455 (basic_scheduler.rs:155)
28  daemon                              0x0000000108c51d28 tokio::runtime::basic_scheduler::enter::_$u7b$$u7b$closure$u7d$$u7d$::hafe141df2711eb4c + 88 (basic_scheduler.rs:213)
29  daemon                              0x0000000108c81cc5 tokio::macros::scoped_tls::ScopedKey$LT$T$GT$::set::hba59ca79a9693ce6 + 165 (scoped_tls.rs:63)
30  daemon                              0x0000000108c51c4d tokio::runtime::basic_scheduler::enter::ha0f9851adeb77824 + 813 (basic_scheduler.rs:213)
31  daemon                              0x0000000108c50fd5 tokio::runtime::basic_scheduler::BasicScheduler$LT$P$GT$::block_on::h680d4f6875d76bd8 + 53 (basic_scheduler.rs:175)
32  daemon                              0x0000000108c4b44b tokio::runtime::Runtime::block_on::_$u7b$$u7b$closure$u7d$$u7d$::had7871dae3fe73e0 + 123 (mod.rs:444)
33  daemon                              0x0000000108c8358f tokio::runtime::context::enter::h247ff58a9477f972 + 111 (context.rs:72)
34  daemon                              0x0000000108c81e94 tokio::runtime::handle::Handle::enter::h029ca8f07dd3c286 + 116 (handle.rs:76)
35  daemon                              0x0000000108c4b3c2 tokio::runtime::Runtime::block_on::h63ea0cd69a874db2 + 82 (mod.rs:448)
36  daemon                              0x0000000108c50217 security_lib::replicator::init_websocket_factory::_$u7b$$u7b$closure$u7d$$u7d$::hefb79f8193a0fc01 + 263 (replicator.rs:132)
37  daemon                              0x0000000108c5bea3 std::sys_common::backtrace::__rust_begin_short_backtrace::h45d644e81b4c481e + 67 (backtrace.rs:137)
38  daemon                              0x0000000108c779a3 std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::he2ad583f98e1ed74 + 67 (mod.rs:465)
39  daemon                              0x0000000108c71133 _$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h10cf58e02b1967fb + 67 (panic.rs:309)
40  daemon                              0x0000000108c4af78 std::panicking::try::do_call::h542c4946c20c31a0 + 104 (panicking.rs:383)
41  daemon                              0x0000000108c4b0ad __rust_try + 29
42  daemon                              0x0000000108c4acc7 std::panicking::try::h245992fcaa6b2a3f + 215 (panicking.rs:345)
43  daemon                              0x0000000108c71213 std::panic::catch_unwind::h2a06603ff7bd49bd + 67 (panic.rs:382)
44  daemon                              0x0000000108c777a0 std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::hde7b0fd6e9e20802 + 320 (mod.rs:463)
45  daemon                              0x0000000108c52ff1 core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h66342c1e0c7d5cab + 17 (function.rs:227)
46  daemon                              0x000000010a35007d std::sys::unix::thread::Thread::new::thread_start::he3e6719579180a65 + 45 (thread.rs:87)
47  libsystem_pthread.dylib             0x00007fff700ec109 _pthread_start + 148
48  libsystem_pthread.dylib             0x00007fff700e7b8b thread_start + 15

It's better clear to investigate

[CedricCouton]

More clues, we have this error at same time :

{C4SocketImpl#73} WebSocket failed to connect! (reason=WebSocket/HTTP status 1000)

[CedricCouton]

Last clues and I think very interesting !

BLIP Connection Opening connection...
BLIP Connection::start
BLIPIO start enqueue(&BLIPIO::_start)
BLIPIO::_start
WebSocket::connect and _delegate = delegate
C4SocketImpl::connect
WebSocketImpl::connect
C4SocketImpl::connect _factory.open
ws_open
Closing with code 1000, msg ''
WebSocket::delegate()
onWebSocketClose
BLIPIO _closed
call c4socket_gotHTTPResponse
Got HTTP response (status 101)
WebSocket::delegate()
[1]    17798 segmentation fault  cargo run

We can see that between the ws_open entry and the call to c4socket_gotHTTPResponse the BLIPIO _closed is called :

void _closed(websocket::CloseStatus status) {
            _onWebSocketMessages(); // process any pending incoming frames

            _webSocket = nullptr;
            if (_connection) {
                Retained<BLIPIO> holdOn (this);
                if (_closingWithError) {
                    status.reason = kException;
                    status.code = _closingWithError->code;
                    status.message = alloc_slice(_closingWithError->what());
                }
                printf("BLIPIO _closed\n");
                _connection->closed(status);
                _connection = nullptr;
                cancelAll(_outbox);
                cancelAll(_icebox);
                cancelAll(_pendingRequests);
                cancelAll(_pendingResponses);
                _requestHandlers.clear();
                release(this); // webSocket is done calling delegate now (balances retain in ctor)
            }
        }

. And so the release(this); which release all transitive pointer I think

[CedricCouton]

We can reproduce the problem with two restart_replicator() successively

[CedricCouton]

I just submitted a PR #55

[Dushistov]

Can you remind me why did you implement your own transport layer and not use the BuiltInWebSocket ?

Funny thing, current master and BuiltInWebSocket have the similar problem:

Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00005629625bb2ce in litecore::blip::Connection::delegate (this=0x0)
    at /home/evgeniy/bigdisk1/projects/rust-infra/couchbase-lite-rust/couchbase-lite-core-sys/couchbase-lite-core/Networking/BLIP/BLIPConnection.hh:58
58              ConnectionDelegate& delegate() const                    {return _delegate;}
[Current thread is 1 (Thread 0x7f07c97e2640 (LWP 195264))]
warning: Missing auto-load script at offset 0 in section .debug_gdb_scripts
of file /home/evgeniy/bigdisk1/projects/rust-infra/couchbase-lite-rust/target/debug/deps/functional_tests-890891bf35f11bdb.
Use `info auto-load python-scripts [REGEXP]' to list them.
(gdb) bt
#0  0x00005629625bb2ce in litecore::blip::Connection::delegate (this=0x0)
    at /home/evgeniy/bigdisk1/projects/rust-infra/couchbase-lite-rust/couchbase-lite-core-sys/couchbase-lite-core/Networking/BLIP/BLIPConnection.hh:58
#1  0x00005629625bfae3 in litecore::blip::Connection::gotHTTPResponse (this=0x0, status=101, headers=...)
    at /home/evgeniy/bigdisk1/projects/rust-infra/couchbase-lite-rust/couchbase-lite-core-sys/couchbase-lite-core/Networking/BLIP/BLIPConnection.cc:688
#2  0x00005629625bc7dd in litecore::blip::BLIPIO::onWebSocketGotHTTPResponse (this=0x7f07d81023e0, status=101, headers=...)
    at /home/evgeniy/bigdisk1/projects/rust-infra/couchbase-lite-rust/couchbase-lite-core-sys/couchbase-lite-core/Networking/BLIP/BLIPConnection.cc:178
#3  0x00005629625d51a2 in litecore::websocket::WebSocketImpl::gotHTTPResponse (this=0x7f07d8034e90, status=101, headersFleece=...)
    at /home/evgeniy/bigdisk1/projects/rust-infra/couchbase-lite-rust/couchbase-lite-core-sys/couchbase-lite-core/Networking/WebSockets/WebSocketImpl.cc:96
#4  0x000056296229dc2e in litecore::websocket::BuiltInWebSocket::_connectLoop (this=0x7f07d8034e90)
    at /home/evgeniy/bigdisk1/projects/rust-infra/couchbase-lite-rust/couchbase-lite-core-sys/couchbase-lite-core/Networking/WebSockets/BuiltInWebSocket.cc:249
#5  0x000056296229cbe5 in litecore::websocket::BuiltInWebSocket::_bgConnect (this=0x7f07d8034e90)
    at /home/evgeniy/bigdisk1/projects/rust-infra/couchbase-lite-rust/couchbase-lite-core-sys/couchbase-lite-core/Networking/WebSockets/BuiltInWebSocket.cc:122
#6  0x00005629622a5db0 in std::__invoke_impl<void, void (litecore::websocket::BuiltInWebSocket::*&)(), litecore::websocket::BuiltInWebSocket*&> (
    __f=@0x7f07d8021a68: (void (litecore::websocket::BuiltInWebSocket::*)(litecore::websocket::BuiltInWebSocket * const)) 0x56296229cb7a <litecore::websocket::BuiltInWebSocket::_bgConnect()>, 
    __t=@0x7f07d8021a78: 0x7f07d8034e90) at /usr/include/c++/11.2.0/bits/invoke.h:74
#7  0x00005629622a5d0d in std::__invoke<void (litecore::websocket::BuiltInWebSocket::*&)(), litecore::websocket::BuiltInWebSocket*&> (
    __fn=@0x7f07d8021a68: (void (litecore::websocket::BuiltInWebSocket::*)(litecore::websocket::BuiltInWebSocket * const)) 0x56296229cb7a <litecore::websocket::BuiltInWebSocket::_bgConnect()>)
    at /usr/include/c++/11.2.0/bits/invoke.h:96
#8  0x00005629622a5c83 in std::_Bind<void (litecore::websocket::BuiltInWebSocket::*(litecore::websocket::BuiltInWebSocket*))()>::__call<void, , 0ul>(std::tuple<>&&, std::_Index_tuple<0ul>) (this=0x7f07d8021a68, 
    __args=...) at /usr/include/c++/11.2.0/functional:420
#9  0x00005629622a5bff in std::_Bind<void (litecore::websocket::BuiltInWebSocket::*(litecore::websocket::BuiltInWebSocket*))()>::operator()<, void>() (this=0x7f07d8021a68)
    at /usr/include/c++/11.2.0/functional:503
#10 0x00005629622a5bae in std::__invoke_impl<void, std::_Bind<void (litecore::websocket::BuiltInWebSocket::*(litecore::websocket::BuiltInWebSocket*))()>>(std::__invoke_other, std::_Bind<void (litecore::websocket::BuiltInWebSocket::*(litecore::websocket::BuiltInWebSocket*))()>&&) (__f=...) at /usr/include/c++/11.2.0/bits/invoke.h:61
#11 0x00005629622a5b71 in std::__invoke<std::_Bind<void (litecore::websocket::BuiltInWebSocket::*(litecore::websocket::BuiltInWebSocket*))()>>(std::_Bind<void (litecore::websocket::BuiltInWebSocket::*(litecore::websocket::BuiltInWebSocket*))()>&&) (__fn=...) at /usr/include/c++/11.2.0/bits/invoke.h:96
#12 0x00005629622a5b1e in std::thread::_Invoker<std::tuple<std::_Bind<void (litecore::websocket::BuiltInWebSocket::*(litecore::websocket::BuiltInWebSocket*))()> > >::_M_invoke<0ul>(std::_Index_tuple<0ul>) (
    this=0x7f07d8021a68) at /usr/include/c++/11.2.0/bits/std_thread.h:253
#13 0x00005629622a59be in std::thread::_Invoker<std::tuple<std::_Bind<void (litecore::websocket::BuiltInWebSocket::*(litecore::websocket::BuiltInWebSocket*))()> > >::operator()() (this=0x7f07d8021a68)
    at /usr/include/c++/11.2.0/bits/std_thread.h:260
#14 0x00005629622a5978 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<std::_Bind<void (litecore::websocket::BuiltInWebSocket::*(litecore::websocket::BuiltInWebSocket*))()> > > >::_M_run() (
    this=0x7f07d8021a60) at /usr/include/c++/11.2.0/bits/std_thread.h:211
#15 0x00007f07e04954d4 in std::execute_native_thread_routine (__p=0x7f07d8021a60) at /usr/src/debug/gcc/libstdc++-v3/src/c++11/thread.cc:82
#16 0x00007f07e01405c2 in start_thread () from /usr/lib/libc.so.6
#17 0x00007f07e01c5584 in clone () from /usr/lib/libc.so.6
(gdb)