pkey: extract common DER/PEM loading code

Add new ossl_pkey_read_raw() function to ossl_pkey.c. This decodes
+data+ using password +pass+ from DER/PEM encoding, and return EVP_PKEY.
The next commit will make use of this - it adds PKCS ruby#8 PrivateKeyInfo
format support.

ext/openssl/ossl_pkey.c

@@ -118,6 +118,34 @@ ossl_pkey_new(EVP_PKEY *pkey)
     return obj;
 }
 
+EVP_PKEY *
+ossl_pkey_read_raw(VALUE data, VALUE pass)
+{
+    BIO *bio;
+    EVP_PKEY *pkey;
+
+    pass = ossl_pem_passwd_value(pass);
+    data = ossl_to_der_if_possible(data);
+    bio = ossl_obj2bio(data);
+
+    pkey = PEM_read_bio_PrivateKey(bio, NULL, ossl_pem_passwd_cb, (void *)pass);
+    if (!pkey) {
+	OSSL_BIO_reset(bio);
+	pkey = PEM_read_bio_PUBKEY(bio, NULL, ossl_pem_passwd_cb, (void *)pass);
+    }
+    if (!pkey) {
+	OSSL_BIO_reset(bio);
+	pkey = d2i_PrivateKey_bio(bio, NULL);
+    }
+    if (!pkey) {
+	OSSL_BIO_reset(bio);
+	pkey = d2i_PUBKEY_bio(bio, NULL);
+    }
+    BIO_free(bio);
+
+    return pkey;
+}
+
 /*
  *  call-seq:
  *     OpenSSL::PKey.read(string [, pwd ] ) -> PKey
@@ -135,25 +163,11 @@ static VALUE
 ossl_pkey_new_from_data(int argc, VALUE *argv, VALUE self)
 {
     EVP_PKEY *pkey;
-    BIO *bio;
     VALUE data, pass;
 
     rb_scan_args(argc, argv, "11", &data, &pass);
-    pass = ossl_pem_passwd_value(pass);
 
-    bio = ossl_obj2bio(data);
-    if (!(pkey = d2i_PrivateKey_bio(bio, NULL))) {
-	OSSL_BIO_reset(bio);
-	if (!(pkey = PEM_read_bio_PrivateKey(bio, NULL, ossl_pem_passwd_cb, (void *)pass))) {
-	    OSSL_BIO_reset(bio);
-	    if (!(pkey = d2i_PUBKEY_bio(bio, NULL))) {
-		OSSL_BIO_reset(bio);
-		pkey = PEM_read_bio_PUBKEY(bio, NULL, ossl_pem_passwd_cb, (void *)pass);
-	    }
-	}
-    }
-
-    BIO_free(bio);
+    pkey = ossl_pkey_read_raw(data, pass);
     if (!pkey)
 	ossl_raise(ePKeyError, "Could not parse PKey");
 

ext/openssl/ossl_pkey.h

@@ -47,7 +47,18 @@ struct ossl_generate_cb_arg {
 int ossl_generate_cb_2(int p, int n, BN_GENCB *cb);
 void ossl_generate_cb_stop(void *ptr);
 
+/*
+ * Creates a new OpenSSL::PKey object from the EVP_PKEY. The EVP_PKEY is not
+ * duplicated.
+ */
 VALUE ossl_pkey_new(EVP_PKEY *);
+/*
+ * Reads DER/PEM string and returns EVP_PKEY. The first argument is the data
+ * (String or IO) and the second is the password (String or nil). Can handle
+ * SubjectPublicKeyInfo, *PrivateKey and PrivateKeyInfo format. Note that it
+ * may leave errors in OpenSSL error queue.
+ */
+EVP_PKEY *ossl_pkey_read_raw(VALUE, VALUE);
 EVP_PKEY *GetPKeyPtr(VALUE);
 EVP_PKEY *DupPKeyPtr(VALUE);
 EVP_PKEY *GetPrivPKeyPtr(VALUE);

ext/openssl/ossl_pkey_dsa.c

@@ -232,33 +232,31 @@ ossl_dsa_initialize(int argc, VALUE *argv, VALUE self)
 	}
     }
     else {
-	pass = ossl_pem_passwd_value(pass);
-	arg = ossl_to_der_if_possible(arg);
-	in = ossl_obj2bio(arg);
-	dsa = PEM_read_bio_DSAPrivateKey(in, NULL, ossl_pem_passwd_cb, (void *)pass);
-	if (!dsa) {
-	    OSSL_BIO_reset(in);
-	    dsa = PEM_read_bio_DSA_PUBKEY(in, NULL, NULL, NULL);
+	EVP_PKEY *tmp_pkey = ossl_pkey_read_raw(arg, pass);
+
+	if (tmp_pkey) {
+	    if (EVP_PKEY_base_id(tmp_pkey) != EVP_PKEY_DSA) {
+		EVP_PKEY_free(tmp_pkey);
+		ossl_raise(eDSAError, "input is not DSA");
+	    }
+	    dsa = EVP_PKEY_get1_DSA(tmp_pkey);
+	    EVP_PKEY_free(tmp_pkey);
 	}
-	if (!dsa) {
-	    OSSL_BIO_reset(in);
-	    dsa = d2i_DSAPrivateKey_bio(in, NULL);
-	}
-	if (!dsa) {
-	    OSSL_BIO_reset(in);
-	    dsa = d2i_DSA_PUBKEY_bio(in, NULL);
-	}
-	if (!dsa) {
-	    OSSL_BIO_reset(in);
+	else {
+	    ossl_clear_error();
+	    pass = ossl_pem_passwd_value(pass);
+	    arg = ossl_to_der_if_possible(arg);
+	    in = ossl_obj2bio(arg);
+
 #define PEM_read_bio_DSAPublicKey(bp,x,cb,u) (DSA *)PEM_ASN1_read_bio( \
-	(d2i_of_void *)d2i_DSAPublicKey, PEM_STRING_DSA_PUBLIC, (bp), (void **)(x), (cb), (u))
+	    (d2i_of_void *)d2i_DSAPublicKey, PEM_STRING_DSA_PUBLIC, (bp), (void **)(x), (cb), (u))
 	    dsa = PEM_read_bio_DSAPublicKey(in, NULL, NULL, NULL);
 #undef PEM_read_bio_DSAPublicKey
-	}
-	BIO_free(in);
-	if (!dsa) {
-	    ossl_clear_error();
-	    ossl_raise(eDSAError, "Neither PUB key nor PRIV key");
+	    BIO_free(in);
+	    if (!dsa) {
+		ossl_clear_error();
+		ossl_raise(eDSAError, "Neither PUB key nor PRIV key");
+	    }
 	}
     }
     if (!EVP_PKEY_assign_DSA(pkey, dsa)) {

ext/openssl/ossl_pkey_ec.c

@@ -248,27 +248,17 @@ static VALUE ossl_ec_key_initialize(int argc, VALUE *argv, VALUE self)
     } else if (rb_obj_is_kind_of(arg, cEC_GROUP)) {
 	ec = ec_key_new_from_group(arg);
     } else {
-	BIO *in;
-
-	pass = ossl_pem_passwd_value(pass);
-	in = ossl_obj2bio(arg);
-
-	ec = PEM_read_bio_ECPrivateKey(in, NULL, ossl_pem_passwd_cb, (void *)pass);
-	if (!ec) {
-	    OSSL_BIO_reset(in);
-	    ec = PEM_read_bio_EC_PUBKEY(in, NULL, ossl_pem_passwd_cb, (void *)pass);
-	}
-	if (!ec) {
-	    OSSL_BIO_reset(in);
-	    ec = d2i_ECPrivateKey_bio(in, NULL);
+	EVP_PKEY *tmp_pkey = ossl_pkey_read_raw(arg, pass);
+
+	if (tmp_pkey) {
+	    if (EVP_PKEY_base_id(tmp_pkey) != EVP_PKEY_EC) {
+		EVP_PKEY_free(tmp_pkey);
+		ossl_raise(eECError, "input is not EC key");
+	    }
+	    ec = EVP_PKEY_get1_EC_KEY(tmp_pkey);
+	    EVP_PKEY_free(tmp_pkey);
 	}
-	if (!ec) {
-	    OSSL_BIO_reset(in);
-	    ec = d2i_EC_PUBKEY_bio(in, NULL);
-	}
-	BIO_free(in);
-
-	if (!ec) {
+	else {
 	    ossl_clear_error();
 	    ec = ec_key_new_from_group(arg);
 	}

ext/openssl/ossl_pkey_rsa.c

@@ -234,33 +234,32 @@ ossl_rsa_initialize(int argc, VALUE *argv, VALUE self)
 	if (!rsa) ossl_raise(eRSAError, NULL);
     }
     else {
-	pass = ossl_pem_passwd_value(pass);
-	arg = ossl_to_der_if_possible(arg);
-	in = ossl_obj2bio(arg);
-	rsa = PEM_read_bio_RSAPrivateKey(in, NULL, ossl_pem_passwd_cb, (void *)pass);
-	if (!rsa) {
-	    OSSL_BIO_reset(in);
-	    rsa = PEM_read_bio_RSA_PUBKEY(in, NULL, NULL, NULL);
-	}
-	if (!rsa) {
-	    OSSL_BIO_reset(in);
-	    rsa = d2i_RSAPrivateKey_bio(in, NULL);
-	}
-	if (!rsa) {
-	    OSSL_BIO_reset(in);
-	    rsa = d2i_RSA_PUBKEY_bio(in, NULL);
+	EVP_PKEY *tmp_pkey = ossl_pkey_read_raw(arg, pass);
+
+	if (tmp_pkey) {
+	    if (EVP_PKEY_base_id(tmp_pkey) != EVP_PKEY_RSA) {
+		EVP_PKEY_free(tmp_pkey);
+		ossl_raise(eRSAError, "input is not RSA");
+	    }
+	    rsa = EVP_PKEY_get1_RSA(tmp_pkey);
+	    EVP_PKEY_free(tmp_pkey);
 	}
-	if (!rsa) {
-	    OSSL_BIO_reset(in);
+	else {
+	    ossl_clear_error();
+	    pass = ossl_pem_passwd_value(pass);
+	    arg = ossl_to_der_if_possible(arg);
+	    in = ossl_obj2bio(arg);
+
 	    rsa = PEM_read_bio_RSAPublicKey(in, NULL, NULL, NULL);
-	}
-	if (!rsa) {
-	    OSSL_BIO_reset(in);
-	    rsa = d2i_RSAPublicKey_bio(in, NULL);
-	}
-	BIO_free(in);
-	if (!rsa) {
-	    ossl_raise(eRSAError, "Neither PUB key nor PRIV key");
+	    if (!rsa) {
+		OSSL_BIO_reset(in);
+		rsa = d2i_RSAPublicKey_bio(in, NULL);
+	    }
+	    BIO_free(in);
+	    if (!rsa) {
+		ossl_clear_error();
+		ossl_raise(eRSAError, "Neither PUB key nor PRIV key");
+	    }
 	}
     }
     if (!EVP_PKEY_assign_RSA(pkey, rsa)) {