nginx 1.0.6 OpenSSL SIGSEGV with AES-NI hardware

[Srebrenko Šehić]

Hi,

nginx (vanilla 1.0.6) will die with a segmentation fault as soon as an
SSL client connects. This is on OpenBSD 4.8, amd64 (OpenSSL 0.9.8k as
distributed by OpenBSD). CPU is Intel(R) Xeon(R) CPU E31240 @ 3.30GHz
where AES is accelerated in hardware.

$ openssl engine -t
(cryptodev) BSD cryptodev engine
     [ available ]
(aesni) Intel AES-NI engine
     [ available ]
(dynamic) Dynamic engine loading support
     [ unavailable ]

Below is the backtrace. If I add "ssl_engine aesni" in the main nginx
config, the problem goes away.

Any clues?

Program received signal SIGSEGV, Segmentation fault.
0x000000020f1d8478 in ?? ()
(gdb) bt
#0  0x000000020f1d8478 in ?? ()
#1  0x0000000205e11f93 in ENGINE_get_cipher (e=0x20e543800, nid=419)
at /usr/src/lib/libssl/crypto/../src/crypto/engine/tb_cipher.c:123
#2  0x0000000205e11c92 in EVP_CipherInit_ex (ctx=0x7f7ffffea480,
cipher=0x206066800, impl=0x20e543800, key=0x2034877d0
"�=\211T\026�~}J1�\1771Q�\027",
    iv=0x7f7ffffea540 "�)#r��\205\0218Ƣ\035�\005w�", enc=Variable
"enc" is not available.
) at /usr/src/lib/libssl/crypto/../src/crypto/evp/enc_min.c:165
#3  0x0000000203d2ac70 in ssl3_send_newsession_ticket (s=0x20fd8ce00)
at /usr/src/lib/libssl/ssl/../src/ssl/s3_srvr.c:2777
#4  0x0000000203d2e07e in ssl3_accept (s=0x20fd8ce00) at
/usr/src/lib/libssl/ssl/../src/ssl/s3_srvr.c:532
#5  0x000000000043f6f1 in ngx_ssl_handshake (c=0x21cf5c6c0) at
src/event/ngx_event_openssl.c:575
#6  0x000000000043fd75 in ngx_ssl_handshake_handler (ev=0x21d8853f0)
at src/event/ngx_event_openssl.c:715
#7  0x000000000043e522 in ngx_kqueue_process_events
(cycle=0x2015c0050, timer=59970, flags=1) at
src/event/modules/ngx_kqueue_module.c:683
#8  0x000000000042f998 in ngx_process_events_and_timers
(cycle=0x2015c0050) at src/event/ngx_event.c:245
#9  0x000000000043c0de in ngx_worker_process_cycle (cycle=0x2015c0050,
data=0x0) at src/os/unix/ngx_process_cycle.c:800
#10 0x00000000004393f5 in ngx_spawn_process (cycle=0x2015c0050,
proc=0x43bf2a <ngx_worker_process_cycle>, data=0x0, name=0x5b7629
"worker process",
    respawn=-3) at src/os/unix/ngx_process.c:196
#11 0x000000000043afdc in ngx_start_worker_processes
(cycle=0x2015c0050, n=1, type=-3) at
src/os/unix/ngx_process_cycle.c:360
#12 0x000000000043a6d5 in ngx_master_process_cycle (cycle=0x2015c0050)
at src/os/unix/ngx_process_cycle.c:136
#13 0x000000000040dcad in main (argc=3, argv=0x7f7ffffeac10) at
src/core/nginx.c:405
(gdb) fr 3
#3  0x0000000203d2ac70 in ssl3_send_newsession_ticket (s=0x20fd8ce00)
at /usr/src/lib/libssl/ssl/../src/ssl/s3_srvr.c:2777
2777                            EVP_EncryptInit_ex(&ctx,
EVP_aes_128_cbc(), NULL,
(gdb) p ctx
$1 = {cipher = 0x0, engine = 0x0, encrypt = 1, buf_len = 0, oiv = '\0'
<repeats 15 times>, iv = '\0' <repeats 15 times>, buf = '\0' <repeats
31 times>,
  num = 0, app_data = 0x0, key_len = 0, flags = 0, cipher_data = 0x0,
final_used = 0, block_mask = 0, final = '\0' <repeats 31 times>}
(gdb) p s
$2 = (SSL *) 0x20fd8ce00
(gdb) p *s
$3 = {version = 769, type = 8192, method = 0x203f51700, rbio =
0x203a02580, wbio = 0x203a02200, bbio = 0x203a02200, rwstate = 1,
in_handshake = 1,
  handshake_func = 0x203d2db70 <ssl3_accept>, server = 1, new_session
= 2, quiet_shutdown = 0, shutdown = 0, state = 8688, rstate = 240,
  init_buf = 0x206db7280, init_msg = 0x20afc9004, init_num = 0,
init_off = 0, packet = 0x20ce8a091 "\026\003\001", packet_length = 0,
s2 = 0x0,
  s3 = 0x20396c000, d1 = 0x0, read_ahead = 1, msg_callback = 0,
msg_callback_arg = 0x0, hit = 0, param = 0x20b92b600, cipher_list =
0x0,
  cipher_list_by_id = 0x0, enc_read_ctx = 0x20ae6c000, read_hash =
0x206080660, expand = 0x0, enc_write_ctx = 0x0, write_hash = 0x0,
compress = 0x0,
  cert = 0x207193800, sid_ctx_length = 4, sid_ctx = "HTTP", '\0'
<repeats 27 times>, session = 0x20fd8ca00, generate_session_id = 0,
verify_mode = 0,
  verify_callback = 0, info_callback = 0, error = 0, error_code = 0,
ctx = 0x203487600, debug = 0, verify_result = 0, ex_data = {sk =
0x206db7bc0,
    dummy = 0}, client_CA = 0x0, references = 1, options = 22547443,
mode = 0, max_cert_list = 102400, first_packet = 0, client_version =
769,
  tlsext_debug_cb = 0, tlsext_debug_arg = 0x0, tlsext_hostname = 0x0,
servername_done = 0, tlsext_status_type = -1, tlsext_status_expected =
0,
  tlsext_ocsp_ids = 0x0, tlsext_ocsp_exts = 0x0, tlsext_ocsp_resp =
0x0, tlsext_ocsp_resplen = -1, tlsext_ticket_expected = 1, initial_ctx
= 0x203487600}
(gdb)