openssl: use SSL_is_server()

* ext/openssl/extconf.rb: Check existence of SSL_is_server(). This
  function was introduced in OpenSSL 1.0.2.
  [ruby-core:75225] [Feature ruby#12324]

* ext/openssl/openssl_missing.h: Implement SSL_is_server() if missing.

* ext/openssl/ossl_ssl.c (ssl_info_cb): Use SSL_is_server() to see if
  the SSL is server. The state machine in OpenSSL was rewritten and
  SSL_get_state() no longer returns SSL_ST_ACCEPT.

  (ossl_ssl_cipher_to_ary, ossl_sslctx_session_get_cb): Add some
  `const`s to suppress warning.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@55289 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

Author: rhenium

ChangeLog

@@ -1,3 +1,18 @@
+Mon Jun  6 01:36:24 2016  Kazuki Yamaguchi  <k@rhe.jp>
+
+	* ext/openssl/extconf.rb: Check existence of SSL_is_server(). This
+	  function was introduced in OpenSSL 1.0.2.
+	  [ruby-core:75225] [Feature #12324]
+
+	* ext/openssl/openssl_missing.h: Implement SSL_is_server() if missing.
+
+	* ext/openssl/ossl_ssl.c (ssl_info_cb): Use SSL_is_server() to see if
+	  the SSL is server. The state machine in OpenSSL was rewritten and
+	  SSL_get_state() no longer returns SSL_ST_ACCEPT.
+
+	  (ossl_ssl_cipher_to_ary, ossl_sslctx_session_get_cb): Add some
+	  `const`s to suppress warning.
+
 Mon Jun  6 01:18:10 2016  Kazuki Yamaguchi  <k@rhe.jp>
 
 	* ext/openssl/ossl_asn1.c (decode_bool): Do the same thing as

ext/openssl/extconf.rb

@@ -114,6 +114,7 @@
 OpenSSL.check_func_or_macro("SSL_CTX_set1_curves_list", "openssl/ssl.h")
 OpenSSL.check_func_or_macro("SSL_CTX_set_ecdh_auto", "openssl/ssl.h")
 OpenSSL.check_func_or_macro("SSL_get_server_tmp_key", "openssl/ssl.h")
+have_func("SSL_is_server")
 
 # added in 1.1.0
 have_func("CRYPTO_lock") || $defs.push("-DHAVE_OPENSSL_110_THREADING_API")

ext/openssl/openssl_missing.h

@@ -61,6 +61,10 @@ int EC_curve_nist2nid(const char *);
 #  define X509_STORE_CTX_get0_store(x) ((x)->ctx)
 #endif
 
+#if !defined(HAVE_SSL_IS_SERVER)
+#  define SSL_is_server(s) ((s)->server)
+#endif
+
 /* added in 1.1.0 */
 #if !defined(HAVE_BN_GENCB_NEW)
 #  define BN_GENCB_new() ((BN_GENCB *)OPENSSL_malloc(sizeof(BN_GENCB)))

ext/openssl/ossl_ssl.c

@@ -342,7 +342,11 @@ ossl_call_session_get_cb(VALUE ary)
 
 /* this method is currently only called for servers (in OpenSSL <= 0.9.8e) */
 static SSL_SESSION *
+#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
+ossl_sslctx_session_get_cb(SSL *ssl, const unsigned char *buf, int len, int *copy)
+#else
 ossl_sslctx_session_get_cb(SSL *ssl, unsigned char *buf, int len, int *copy)
+#endif
 {
     VALUE ary, ssl_obj, ret_obj;
     SSL_SESSION *sess;
@@ -650,15 +654,13 @@ ssl_alpn_select_cb(SSL *ssl, const unsigned char **out, unsigned char *outlen, c
 #endif
 #endif /* HAVE_SSL_CTX_SET_NEXT_PROTO_SELECT_CB || HAVE_SSL_CTX_SET_ALPN_SELECT_CB */
 
-/* This function may serve as the entry point to support further
- * callbacks. */
+/* This function may serve as the entry point to support further callbacks. */
 static void
 ssl_info_cb(const SSL *ssl, int where, int val)
 {
-    int state = SSL_state(ssl);
+    int is_server = SSL_is_server((SSL *)ssl);
 
-    if ((where & SSL_CB_HANDSHAKE_START) &&
-	(state & SSL_ST_ACCEPT)) {
+    if (is_server && where & SSL_CB_HANDSHAKE_START) {
 	ssl_renegotiation_cb(ssl);
     }
 }
@@ -887,7 +889,7 @@ ossl_sslctx_setup(VALUE self)
 }
 
 static VALUE
-ossl_ssl_cipher_to_ary(SSL_CIPHER *cipher)
+ossl_ssl_cipher_to_ary(const SSL_CIPHER *cipher)
 {
     VALUE ary;
     int bits, alg_bits;
@@ -913,7 +915,7 @@ ossl_sslctx_get_ciphers(VALUE self)
 {
     SSL_CTX *ctx;
     STACK_OF(SSL_CIPHER) *ciphers;
-    SSL_CIPHER *cipher;
+    const SSL_CIPHER *cipher;
     VALUE ary;
     int i, num;