From 14ee87d793362809d97533a8d905eaab0954f1ff Mon Sep 17 00:00:00 2001 From: Pankaj Garg Date: Tue, 12 Jul 2022 13:34:37 -0700 Subject: [PATCH] Move QUIC TLS certificate code in its own file (#26569) --- Cargo.lock | 2 - client/Cargo.toml | 2 - client/src/connection_cache.rs | 84 +++++------------------- client/src/nonblocking/quic_client.rs | 19 ++++-- programs/bpf/Cargo.lock | 2 - streamer/src/lib.rs | 1 + streamer/src/nonblocking/quic.rs | 30 ++++----- streamer/src/quic.rs | 68 +------------------- streamer/src/tls_certificates.rs | 92 +++++++++++++++++++++++++++ 9 files changed, 134 insertions(+), 166 deletions(-) create mode 100644 streamer/src/tls_certificates.rs diff --git a/Cargo.lock b/Cargo.lock index a5bee8eac4..2e38b21024 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4948,13 +4948,11 @@ dependencies = [ "jsonrpc-http-server", "lazy_static", "log", - "pkcs8", "quinn", "quinn-proto", "rand 0.7.3", "rand_chacha 0.2.2", "rayon", - "rcgen", "reqwest", "rustls 0.20.6", "semver 1.0.10", diff --git a/client/Cargo.toml b/client/Cargo.toml index c42fe68687..310347ef52 100644 --- a/client/Cargo.toml +++ b/client/Cargo.toml @@ -27,13 +27,11 @@ itertools = "0.10.2" jsonrpc-core = "18.0.0" lazy_static = "1.4.0" log = "0.4.17" -pkcs8 = { version = "0.8.0", features = ["alloc"] } quinn = "0.8.3" quinn-proto = "0.8.3" rand = "0.7.0" rand_chacha = "0.2.2" rayon = "1.5.3" -rcgen = "0.9.2" reqwest = { version = "0.11.11", default-features = false, features = ["blocking", "brotli", "deflate", "gzip", "rustls-tls", "json"] } rustls = { version = "0.20.6", features = ["dangerous_configuration"] } semver = "1.0.10" diff --git a/client/src/connection_cache.rs b/client/src/connection_cache.rs index 8dad17fc9a..82f516feb9 100644 --- a/client/src/connection_cache.rs +++ b/client/src/connection_cache.rs @@ -7,11 +7,10 @@ use { tpu_connection::{BlockingConnection, ClientStats}, }, indexmap::map::{Entry, IndexMap}, - pkcs8::{der::Document, AlgorithmIdentifier, ObjectIdentifier}, rand::{thread_rng, Rng}, - rcgen::{CertificateParams, DistinguishedName, DnType, SanType}, solana_measure::measure::Measure, solana_sdk::{quic::QUIC_PORT_OFFSET, signature::Keypair, timing::AtomicInterval}, + solana_streamer::tls_certificates::new_self_signed_tls_certificate_chain, std::{ error::Error, net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, @@ -255,68 +254,6 @@ impl ConnectionPool { } } -pub(crate) fn new_cert( - identity_keypair: &Keypair, - san: IpAddr, -) -> Result> { - // Generate a self-signed cert from client's identity key - let cert_params = new_cert_params(identity_keypair, san); - let cert = rcgen::Certificate::from_params(cert_params)?; - let cert_der = cert.serialize_der().unwrap(); - let priv_key = cert.serialize_private_key_der(); - let priv_key = rustls::PrivateKey(priv_key); - Ok(QuicClientCertificate { - certificates: vec![rustls::Certificate(cert_der)], - key: priv_key, - }) -} - -fn convert_to_rcgen_keypair(identity_keypair: &Keypair) -> rcgen::KeyPair { - // from https://datatracker.ietf.org/doc/html/rfc8410#section-3 - const ED25519_IDENTIFIER: [u32; 4] = [1, 3, 101, 112]; - let mut private_key = Vec::::with_capacity(34); - private_key.extend_from_slice(&[0x04, 0x20]); // ASN.1 OCTET STRING - private_key.extend_from_slice(identity_keypair.secret().as_bytes()); - let key_pkcs8 = pkcs8::PrivateKeyInfo { - algorithm: AlgorithmIdentifier { - oid: ObjectIdentifier::from_arcs(&ED25519_IDENTIFIER).unwrap(), - parameters: None, - }, - private_key: &private_key, - public_key: None, - }; - let key_pkcs8_der = key_pkcs8 - .to_der() - .expect("Failed to convert keypair to DER") - .to_der(); - - // Parse private key into rcgen::KeyPair struct. - rcgen::KeyPair::from_der(&key_pkcs8_der).expect("Failed to parse keypair from DER") -} - -fn new_cert_params(identity_keypair: &Keypair, san: IpAddr) -> CertificateParams { - // TODO(terorie): Is it safe to sign the TLS cert with the identity private key? - - // Unfortunately, rcgen does not accept a "raw" Ed25519 key. - // We have to convert it to DER and pass it to the library. - - // Convert private key into PKCS#8 v1 object. - // RFC 8410, Section 7: Private Key Format - // https://datatracker.ietf.org/doc/html/rfc8410#section- - - let keypair = convert_to_rcgen_keypair(identity_keypair); - - let mut cert_params = CertificateParams::default(); - cert_params.subject_alt_names = vec![SanType::IpAddress(san)]; - cert_params.alg = &rcgen::PKCS_ED25519; - cert_params.key_pair = Some(keypair); - cert_params.distinguished_name = DistinguishedName::new(); - cert_params - .distinguished_name - .push(DnType::CommonName, "Solana node"); - cert_params -} - impl ConnectionCache { pub fn new(connection_pool_size: usize) -> Self { // The minimum pool size is 1. @@ -333,8 +270,11 @@ impl ConnectionCache { keypair: &Keypair, ipaddr: IpAddr, ) -> Result<(), Box> { - let cert = new_cert(keypair, ipaddr)?; - self.client_certificate = Arc::new(cert); + let (certs, priv_key) = new_self_signed_tls_certificate_chain(keypair, ipaddr)?; + self.client_certificate = Arc::new(QuicClientCertificate { + certificates: certs, + key: priv_key, + }); Ok(()) } @@ -566,6 +506,11 @@ impl ConnectionCache { impl Default for ConnectionCache { fn default() -> Self { + let (certs, priv_key) = new_self_signed_tls_certificate_chain( + &Keypair::new(), + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to initialize QUIC client certificates"); Self { map: RwLock::new(IndexMap::with_capacity(MAX_CONNECTIONS)), stats: Arc::new(ConnectionCacheStats::default()), @@ -577,9 +522,10 @@ impl Default for ConnectionCache { .expect("Unable to bind to UDP socket"), ) }), - client_certificate: new_cert(&Keypair::new(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))) - .map(Arc::new) - .expect("Failed to initialize QUIC client certificates"), + client_certificate: Arc::new(QuicClientCertificate { + certificates: certs, + key: priv_key, + }), } } } diff --git a/client/src/nonblocking/quic_client.rs b/client/src/nonblocking/quic_client.rs index c9a8caeafb..c3b769c286 100644 --- a/client/src/nonblocking/quic_client.rs +++ b/client/src/nonblocking/quic_client.rs @@ -3,10 +3,8 @@ //! server's flow control. use { crate::{ - client_error::ClientErrorKind, - connection_cache::{new_cert, ConnectionCacheStats}, - nonblocking::tpu_connection::TpuConnection, - tpu_connection::ClientStats, + client_error::ClientErrorKind, connection_cache::ConnectionCacheStats, + nonblocking::tpu_connection::TpuConnection, tpu_connection::ClientStats, }, async_mutex::Mutex, async_trait::async_trait, @@ -27,6 +25,7 @@ use { signature::Keypair, transport::Result as TransportResult, }, + solana_streamer::tls_certificates::new_self_signed_tls_certificate_chain, std::{ net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, sync::{atomic::Ordering, Arc}, @@ -134,9 +133,15 @@ impl QuicLazyInitializedEndpoint { impl Default for QuicLazyInitializedEndpoint { fn default() -> Self { - let certificate = new_cert(&Keypair::new(), IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))) - .expect("Failed to create QUIC client certificate"); - Self::new(Arc::new(certificate)) + let (certs, priv_key) = new_self_signed_tls_certificate_chain( + &Keypair::new(), + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to create QUIC client certificate"); + Self::new(Arc::new(QuicClientCertificate { + certificates: certs, + key: priv_key, + })) } } diff --git a/programs/bpf/Cargo.lock b/programs/bpf/Cargo.lock index feb6619af3..22e991228d 100644 --- a/programs/bpf/Cargo.lock +++ b/programs/bpf/Cargo.lock @@ -4593,13 +4593,11 @@ dependencies = [ "jsonrpc-core", "lazy_static", "log", - "pkcs8", "quinn", "quinn-proto", "rand 0.7.3", "rand_chacha 0.2.2", "rayon", - "rcgen", "reqwest", "rustls 0.20.6", "semver", diff --git a/streamer/src/lib.rs b/streamer/src/lib.rs index 22d14b5932..2dfee737cd 100644 --- a/streamer/src/lib.rs +++ b/streamer/src/lib.rs @@ -6,6 +6,7 @@ pub mod recvmmsg; pub mod sendmmsg; pub mod socket; pub mod streamer; +pub mod tls_certificates; #[macro_use] extern crate log; diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index 679eb0d7c8..42bc7f9ad7 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -2,6 +2,7 @@ use { crate::{ quic::{configure_server, QuicServerError, StreamStats}, streamer::StakedNodes, + tls_certificates::get_pubkey_from_tls_certificate, }, crossbeam_channel::Sender, futures_util::stream::StreamExt, @@ -15,7 +16,6 @@ use { solana_perf::packet::PacketBatch, solana_sdk::{ packet::{Packet, PACKET_DATA_SIZE}, - pubkey::Pubkey, quic::{QUIC_CONNECTION_HANDSHAKE_TIMEOUT_MS, QUIC_MAX_UNSTAKED_CONCURRENT_STREAMS}, signature::Keypair, timing, @@ -32,7 +32,6 @@ use { task::JoinHandle, time::{sleep, timeout}, }, - x509_parser::{prelude::*, public_key::PublicKey}, }; const QUIC_TOTAL_STAKED_CONCURRENT_STREAMS: f64 = 100_000f64; @@ -139,21 +138,11 @@ fn get_connection_stake(connection: &Connection, staked_nodes: Arc>().ok()) .and_then(|der_certs| { - der_certs.first().and_then(|der_cert| { - X509Certificate::from_der(der_cert.as_ref()) - .ok() - .and_then(|(_, cert)| { - cert.public_key().parsed().ok().and_then(|key| match key { - PublicKey::Unknown(inner_key) => { - let pubkey = Pubkey::new(inner_key); - debug!("Peer public key is {:?}", pubkey); + get_pubkey_from_tls_certificate(&der_certs).and_then(|pubkey| { + debug!("Peer public key is {:?}", pubkey); - let staked_nodes = staked_nodes.read().unwrap(); - staked_nodes.pubkey_stake_map.get(&pubkey).copied() - } - _ => None, - }) - }) + let staked_nodes = staked_nodes.read().unwrap(); + staked_nodes.pubkey_stake_map.get(&pubkey).copied() }) }) .unwrap_or(0) @@ -657,7 +646,10 @@ impl ConnectionTable { pub mod test { use { super::*, - crate::quic::{new_cert, MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS}, + crate::{ + quic::{MAX_STAKED_CONNECTIONS, MAX_UNSTAKED_CONNECTIONS}, + tls_certificates::new_self_signed_tls_certificate_chain, + }, crossbeam_channel::{unbounded, Receiver}, quinn::{ClientConfig, IdleTimeout, VarInt}, solana_sdk::{ @@ -693,8 +685,8 @@ pub mod test { pub fn get_client_config(keypair: &Keypair) -> ClientConfig { let ipaddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); - let (certs, key) = - new_cert(keypair, ipaddr).expect("Failed to generate client certificate"); + let (certs, key) = new_self_signed_tls_certificate_chain(keypair, ipaddr) + .expect("Failed to generate client certificate"); let mut crypto = rustls::ClientConfig::builder() .with_safe_defaults() diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index c5fef2a5e9..2aef26c8e1 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -1,10 +1,8 @@ use { - crate::streamer::StakedNodes, + crate::{streamer::StakedNodes, tls_certificates::new_self_signed_tls_certificate_chain}, crossbeam_channel::Sender, pem::Pem, - pkcs8::{der::Document, AlgorithmIdentifier, ObjectIdentifier}, quinn::{IdleTimeout, ServerConfig, VarInt}, - rcgen::{CertificateParams, DistinguishedName, DnType, SanType}, rustls::{server::ClientCertVerified, Certificate, DistinguishedNames}, solana_perf::packet::PacketBatch, solana_sdk::{ @@ -13,7 +11,6 @@ use { signature::Keypair, }, std::{ - error::Error, net::{IpAddr, UdpSocket}, sync::{ atomic::{AtomicBool, AtomicUsize, Ordering}, @@ -59,7 +56,8 @@ pub(crate) fn configure_server( gossip_host: IpAddr, ) -> Result<(ServerConfig, String), QuicServerError> { let (cert_chain, priv_key) = - new_cert(identity_keypair, gossip_host).map_err(|_e| QuicServerError::ConfigureFailed)?; + new_self_signed_tls_certificate_chain(identity_keypair, gossip_host) + .map_err(|_e| QuicServerError::ConfigureFailed)?; let cert_chain_pem_parts: Vec = cert_chain .iter() .map(|cert| Pem { @@ -94,66 +92,6 @@ pub(crate) fn configure_server( Ok((server_config, cert_chain_pem)) } -pub(crate) fn new_cert( - identity_keypair: &Keypair, - san: IpAddr, -) -> Result<(Vec, rustls::PrivateKey), Box> { - // Generate a self-signed cert from validator identity key - let cert_params = new_cert_params(identity_keypair, san); - let cert = rcgen::Certificate::from_params(cert_params)?; - let cert_der = cert.serialize_der().unwrap(); - let priv_key = cert.serialize_private_key_der(); - let priv_key = rustls::PrivateKey(priv_key); - let cert_chain = vec![rustls::Certificate(cert_der)]; - Ok((cert_chain, priv_key)) -} - -fn convert_to_rcgen_keypair(identity_keypair: &Keypair) -> rcgen::KeyPair { - // from https://datatracker.ietf.org/doc/html/rfc8410#section-3 - const ED25519_IDENTIFIER: [u32; 4] = [1, 3, 101, 112]; - let mut private_key = Vec::::with_capacity(34); - private_key.extend_from_slice(&[0x04, 0x20]); // ASN.1 OCTET STRING - private_key.extend_from_slice(identity_keypair.secret().as_bytes()); - let key_pkcs8 = pkcs8::PrivateKeyInfo { - algorithm: AlgorithmIdentifier { - oid: ObjectIdentifier::from_arcs(&ED25519_IDENTIFIER).unwrap(), - parameters: None, - }, - private_key: &private_key, - public_key: None, - }; - let key_pkcs8_der = key_pkcs8 - .to_der() - .expect("Failed to convert keypair to DER") - .to_der(); - - // Parse private key into rcgen::KeyPair struct. - rcgen::KeyPair::from_der(&key_pkcs8_der).expect("Failed to parse keypair from DER") -} - -fn new_cert_params(identity_keypair: &Keypair, san: IpAddr) -> CertificateParams { - // TODO(terorie): Is it safe to sign the TLS cert with the identity private key? - - // Unfortunately, rcgen does not accept a "raw" Ed25519 key. - // We have to convert it to DER and pass it to the library. - - // Convert private key into PKCS#8 v1 object. - // RFC 8410, Section 7: Private Key Format - // https://datatracker.ietf.org/doc/html/rfc8410#section- - - let keypair = convert_to_rcgen_keypair(identity_keypair); - - let mut cert_params = CertificateParams::default(); - cert_params.subject_alt_names = vec![SanType::IpAddress(san)]; - cert_params.alg = &rcgen::PKCS_ED25519; - cert_params.key_pair = Some(keypair); - cert_params.distinguished_name = DistinguishedName::new(); - cert_params - .distinguished_name - .push(DnType::CommonName, "Solana node"); - cert_params -} - fn rt() -> Runtime { Builder::new_multi_thread() .worker_threads(NUM_QUIC_STREAMER_WORKER_THREADS) diff --git a/streamer/src/tls_certificates.rs b/streamer/src/tls_certificates.rs new file mode 100644 index 0000000000..27d3365c88 --- /dev/null +++ b/streamer/src/tls_certificates.rs @@ -0,0 +1,92 @@ +use { + pkcs8::{der::Document, AlgorithmIdentifier, ObjectIdentifier}, + rcgen::{CertificateParams, DistinguishedName, DnType, SanType}, + solana_sdk::{pubkey::Pubkey, signature::Keypair}, + std::{error::Error, net::IpAddr}, + x509_parser::{prelude::*, public_key::PublicKey}, +}; + +pub fn new_self_signed_tls_certificate_chain( + keypair: &Keypair, + san: IpAddr, +) -> Result<(Vec, rustls::PrivateKey), Box> { + // TODO(terorie): Is it safe to sign the TLS cert with the identity private key? + + // Unfortunately, rcgen does not accept a "raw" Ed25519 key. + // We have to convert it to DER and pass it to the library. + + // Convert private key into PKCS#8 v1 object. + // RFC 8410, Section 7: Private Key Format + // https://datatracker.ietf.org/doc/html/rfc8410#section- + + // from https://datatracker.ietf.org/doc/html/rfc8410#section-3 + const ED25519_IDENTIFIER: [u32; 4] = [1, 3, 101, 112]; + let mut private_key = Vec::::with_capacity(34); + private_key.extend_from_slice(&[0x04, 0x20]); // ASN.1 OCTET STRING + private_key.extend_from_slice(keypair.secret().as_bytes()); + let key_pkcs8 = pkcs8::PrivateKeyInfo { + algorithm: AlgorithmIdentifier { + oid: ObjectIdentifier::from_arcs(&ED25519_IDENTIFIER).expect("Failed to convert OID"), + parameters: None, + }, + private_key: &private_key, + public_key: None, + }; + let key_pkcs8_der = key_pkcs8 + .to_der() + .expect("Failed to convert keypair to DER") + .to_der(); + + let rcgen_keypair = rcgen::KeyPair::from_der(&key_pkcs8_der)?; + + let mut cert_params = CertificateParams::default(); + cert_params.subject_alt_names = vec![SanType::IpAddress(san)]; + cert_params.alg = &rcgen::PKCS_ED25519; + cert_params.key_pair = Some(rcgen_keypair); + cert_params.distinguished_name = DistinguishedName::new(); + cert_params + .distinguished_name + .push(DnType::CommonName, "Solana node"); + + let cert = rcgen::Certificate::from_params(cert_params)?; + let cert_der = cert.serialize_der().unwrap(); + let priv_key = cert.serialize_private_key_der(); + let priv_key = rustls::PrivateKey(priv_key); + let cert_chain = vec![rustls::Certificate(cert_der)]; + Ok((cert_chain, priv_key)) +} + +pub fn get_pubkey_from_tls_certificate(certificates: &[rustls::Certificate]) -> Option { + certificates.first().and_then(|der_cert| { + X509Certificate::from_der(der_cert.as_ref()) + .ok() + .and_then(|(_, cert)| { + cert.public_key().parsed().ok().and_then(|key| match key { + PublicKey::Unknown(inner_key) => Some(Pubkey::new(inner_key)), + _ => None, + }) + }) + }) +} + +#[cfg(test)] +mod tests { + use {super::*, solana_sdk::signer::Signer, std::net::Ipv4Addr}; + + #[test] + fn test_generate_tls_certificate() { + let keypair = Keypair::new(); + + if let Ok((certs, _)) = + new_self_signed_tls_certificate_chain(&keypair, IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))) + { + if let Some(pubkey) = get_pubkey_from_tls_certificate(&certs) { + assert_eq!(pubkey, keypair.pubkey()); + } else { + panic!("Failed to get certificate pubkey"); + } + } else { + panic!("Failed to generate certificates"); + } + } +}