protoc gets stuck

[onsails]

When compiling my project after clean, sbt gets stuck. After killing sbt and re-running sbt compile it succeeds. If I clean and compile again, I get stalled compilation фпфшт. It happens on both OS X and Ubuntu.

Using thread dump and strace debug I found out that sbt indefinitely waits for process /tmp/protoc7087999790195032730.exe --plugin=protoc-gen-scala=/tmp/scalapbgen3078536628637421129 -I/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/src/main/protobuf -I/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/target/protobuf_external -I/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-models/src/main/protobuf -I/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-core/src/main/protobuf --scala_out=:/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/target/src_managed/main/compiled_protobuf /home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/src/main/protobuf/bot.proto which never completes.

bot.proto looks pretty simple:

syntax = 'proto3';

package im.actor.server;

message BotWebHook {
    int32 user_id = 1;
    string name = 2;
}

[onsails]

Here is strace for protoc:

$ strace /tmp/protoc7087999790195032730.exe --plugin=protoc-gen-scala=/tmp/scalapbgen3078536628637421129 -I/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/src/main/protobuf -I/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/target/protobuf_external -I/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-models/src/main/protobuf -I/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-core/src/main/protobuf --scala_out=:/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/target/src_managed/main/compiled_protobuf /home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/src/main/protobuf/bot.proto
execve("/tmp/protoc7087999790195032730.exe", ["/tmp/protoc7087999790195032730.e"..., "--plugin=protoc-gen-scala=/tmp/s"..., "-I/home/ubuntu/TeamCity/buildAge"..., "-I/home/ubuntu/TeamCity/buildAge"..., "-I/home/ubuntu/TeamCity/buildAge"..., "-I/home/ubuntu/TeamCity/buildAge"..., "--scala_out=:/home/ubuntu/TeamCi"..., "/home/ubuntu/TeamCity/buildAgent"...], [/* 26 vars */]) = 0
brk(0)                                  = 0x1cd8000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa192cce000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=24147, ...}) = 0
mmap(NULL, 24147, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fa192cc8000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0po\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=141574, ...}) = 0
mmap(NULL, 2217264, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa192890000
mprotect(0x7fa1928a9000, 2093056, PROT_NONE) = 0
mmap(0x7fa192aa8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18000) = 0x7fa192aa8000
mmap(0x7fa192aaa000, 13616, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa192aaa000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20V\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=1071552, ...}) = 0
mmap(NULL, 3166568, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa19258a000
mprotect(0x7fa19268f000, 2093056, PROT_NONE) = 0
mmap(0x7fa19288e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x104000) = 0x7fa19288e000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\37\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1840928, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa192cc7000
mmap(NULL, 3949248, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fa1921c5000
mprotect(0x7fa192380000, 2093056, PROT_NONE) = 0
mmap(0x7fa19257f000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1ba000) = 0x7fa19257f000
mmap(0x7fa192585000, 17088, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fa192585000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa192cc6000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fa192cc4000
arch_prctl(ARCH_SET_FS, 0x7fa192cc4780) = 0
mprotect(0x7fa19257f000, 16384, PROT_READ) = 0
mprotect(0x7fa19288e000, 4096, PROT_READ) = 0
mprotect(0x7fa192aa8000, 4096, PROT_READ) = 0
mprotect(0x7fa192cd0000, 4096, PROT_READ) = 0
munmap(0x7fa192cc8000, 24147)           = 0
set_tid_address(0x7fa192cc4a50)         = 5143
set_robust_list(0x7fa192cc4a60, 24)     = 0
futex(0x7ffe53de9df0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, NULL, 7fa192cc4780) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0x7fa1928969f0, [], SA_RESTORER|SA_SIGINFO, 0x7fa1928a0340}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7fa192896a80, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7fa1928a0340}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
futex(0x9ab61c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x9ab628, FUTEX_WAKE_PRIVATE, 2147483647) = 0
brk(0)                                  = 0x1cd8000
brk(0x1cf9000)                          = 0x1cf9000
access("/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/src/main/protobuf", F_OK) = 0
access("/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/target/protobuf_external", F_OK) = 0
access("/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-models/src/main/protobuf", F_OK) = 0
access("/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-core/src/main/protobuf", F_OK) = 0
readlink("/proc/self/exe", "/tmp/protoc7087999790195032730.e"..., 4096) = 34
access("/tmp/google/protobuf/descriptor.proto", F_OK) = -1 ENOENT (No such file or directory)
access("/tmp/include/google/protobuf/descriptor.proto", F_OK) = -1 ENOENT (No such file or directory)
open("/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/src/main/protobuf/bot.proto", O_RDONLY) = 3
close(3)                                = 0
open("/home/ubuntu/TeamCity/buildAgent/work/3a46fa3b0b70b255/actor-server/actor-bots/src/main/protobuf/bot.proto", O_RDONLY) = 3
read(3, "syntax = 'proto3';\n\npackage im.a"..., 8192) = 113
read(3, "", 8192)                       = 0
close(3)                                = 0
pipe([3, 4])                            = 0
pipe([5, 6])                            = 0
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7fa192cc4a50) = 5144
close(3)                                = 0
close(6)                                = 0
rt_sigaction(SIGPIPE, {SIG_IGN, [PIPE], SA_RESTORER|SA_RESTART, 0x7fa1921fbd40}, {SIG_DFL, [], 0}, 8) = 0
select(6, [5], [4], NULL, NULL)         = 1 (out [4])
write(4, "\n\tbot.protoz\224\2\n\tbot.proto\22\17im.ac"..., 290) = 290
close(4)                                = 0
select(6, [5], [], NULL, NULL

[onsails]

lsof does not show anything for fd 6

$ sudo lsof -p 5143
COMMAND    PID   USER   FD   TYPE DEVICE SIZE/OFF    NODE NAME
protoc708 5143 ubuntu  cwd    DIR  202,1     4096 2097153 /home/ubuntu/TeamCity
protoc708 5143 ubuntu  rtd    DIR  202,1     4096       2 /
protoc708 5143 ubuntu  txt    REG  202,1  3770744    5676 /tmp/protoc7087999790195032730.exe
protoc708 5143 ubuntu  mem    REG  202,1  1840928  397614 /lib/x86_64-linux-gnu/libc-2.19.so
protoc708 5143 ubuntu  mem    REG  202,1  1071552  397599 /lib/x86_64-linux-gnu/libm-2.19.so
protoc708 5143 ubuntu  mem    REG  202,1   141574  397615 /lib/x86_64-linux-gnu/libpthread-2.19.so
protoc708 5143 ubuntu  mem    REG  202,1   149120  397611 /lib/x86_64-linux-gnu/ld-2.19.so
protoc708 5143 ubuntu    0u   CHR  136,0      0t0       3 /dev/pts/0
protoc708 5143 ubuntu    1u   CHR  136,0      0t0       3 /dev/pts/0
protoc708 5143 ubuntu    2u   CHR  136,0      0t0       3 /dev/pts/0
protoc708 5143 ubuntu    5r  FIFO    0,8      0t0   56398 pipe

[onsails]

Can't it be a bug with scalapbgen plugin?

[onsails]

Worth noting that ~ one from 10 attempts compilation succeeds without any problems.

[thesamet]

@prettynatty thanks for the detailed report. I was able to reproduce this bug. Can you try upgrading to sbt-scalapb 0.5.17 in actor-server/project/plugins.sbt.

I submitted scalapb/ScalaPB@a9459ba which makes the input and output go through two named pipes instead of reusing a single one. Please let me know if it work or if it doesn't work for you.

[onsails]

Looks like it works well now, thanks!

[onsails]

After migrating to 0.5.21 from 0.5.17 issue appeared again.
No strace info yet because I can't reproduce it on linux, only on OSX where dtrace is not working without some magic in safe mode. If you need it, I'll provide later.

[thesamet]

@prettynatty which protoc version you use? Can you send a minimal example on how to reproduce? To get a JVM stack trace, hit Ctrl \