Handle cargo registry index lookup requests (#33681)

Cargo.lock

@@ -2209,21 +2209,6 @@ version = "0.27.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
 
-[[package]]
-name = "git2"
-version = "0.18.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd"
-dependencies = [
- "bitflags 2.3.3",
- "libc",
- "libgit2-sys",
- "log",
- "openssl-probe",
- "openssl-sys",
- "url 2.4.1",
-]
-
 [[package]]
 name = "glob"
 version = "0.3.0"
@@ -2462,12 +2447,6 @@ dependencies = [
  "pin-project-lite",
 ]
 
-[[package]]
-name = "http-range"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21dec9db110f5f872ed9699c3ecf50cf16f423502706ba5c72462e28d3157573"
-
 [[package]]
 name = "httparse"
 version = "1.8.0"
@@ -2542,25 +2521,6 @@ dependencies = [
  "tokio-rustls",
 ]
 
-[[package]]
-name = "hyper-staticfile"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "318ca89e4827e7fe4ddd2824f52337239796ae8ecc761a663324407dc3d8d7e7"
-dependencies = [
- "futures-util",
- "http",
- "http-range",
- "httpdate",
- "hyper",
- "mime_guess",
- "percent-encoding 2.3.0",
- "rand 0.8.5",
- "tokio",
- "url 2.4.1",
- "winapi 0.3.9",
-]
-
 [[package]]
 name = "hyper-timeout"
 version = "0.4.1"
@@ -2912,20 +2872,6 @@ version = "0.2.149"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
 
-[[package]]
-name = "libgit2-sys"
-version = "0.16.1+1.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2a2bb3680b094add03bb3732ec520ece34da31a8cd2d633d1389d0f0fb60d0c"
-dependencies = [
- "cc",
- "libc",
- "libssh2-sys",
- "libz-sys",
- "openssl-sys",
- "pkg-config",
-]
-
 [[package]]
 name = "libloading"
 version = "0.7.4"
@@ -3005,20 +2951,6 @@ dependencies = [
  "libsecp256k1-core",
 ]
 
-[[package]]
-name = "libssh2-sys"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dc8a030b787e2119a731f1951d6a773e2280c660f8ec4b0f5e1505a386e71ee"
-dependencies = [
- "cc",
- "libc",
- "libz-sys",
- "openssl-sys",
- "pkg-config",
- "vcpkg",
-]
-
 [[package]]
 name = "libz-sys"
 version = "1.1.3"
@@ -3026,7 +2958,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66"
 dependencies = [
  "cc",
- "libc",
  "pkg-config",
  "vcpkg",
 ]
@@ -3182,16 +3113,6 @@ version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
-[[package]]
-name = "mime_guess"
-version = "2.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef"
-dependencies = [
- "mime",
- "unicase",
-]
-
 [[package]]
 name = "min-max-heap"
 version = "1.3.0"
@@ -5565,13 +5486,12 @@ version = "1.18.0"
 dependencies = [
  "clap 2.33.3",
  "flate2",
- "git2",
  "hyper",
- "hyper-staticfile",
  "log",
  "rustc_version 0.4.0",
  "serde",
  "serde_json",
+ "sha2 0.10.8",
  "solana-clap-utils",
  "solana-cli",
  "solana-cli-config",

Cargo.toml

@@ -206,7 +206,6 @@ gag = "1.0.0"
 generic-array = { version = "0.14.7", default-features = false }
 gethostname = "0.2.3"
 getrandom = "0.2.10"
-git2 = "0.18.1"
 goauth = "0.13.1"
 hex = "0.4.3"
 hidapi = { version = "2.4.1", default-features = false }
@@ -216,7 +215,6 @@ http = "0.2.9"
 humantime = "2.0.1"
 hyper = "0.14.27"
 hyper-proxy = "0.9.1"
-hyper-staticfile = "0.9.5"
 im = "15.1.0"
 index_list = "0.2.7"
 indexmap = "2.0.2"

cargo-registry/Cargo.toml

@@ -12,12 +12,11 @@ edition = { workspace = true }
 [dependencies]
 clap = { workspace = true }
 flate2 = { workspace = true }
-git2 = { workspace = true }
 hyper = { workspace = true, features = ["full"] }
-hyper-staticfile = { workspace = true }
 log = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
+sha2 = { workspace = true }
 solana-clap-utils = { workspace = true }
 solana-cli = { workspace = true }
 solana-cli-config = { workspace = true }

cargo-registry/src/main.rs

@@ -3,6 +3,7 @@ use {
     crate::{
         client::Client,
         publisher::{Error, Publisher},
+        sparse_index::RegistryIndex,
     },
     hyper::{
         body,
@@ -30,15 +31,17 @@ impl CargoRegistryService {
     async fn handle_publish_request(
         request: hyper::Request<hyper::Body>,
         client: Arc<Client>,
+        index: Arc<RegistryIndex>,
     ) -> hyper::Response<hyper::Body> {
         info!("Handling request to publish the crate");
         let bytes = body::to_bytes(request.into_body()).await;
 
         match bytes {
             Ok(data) => {
-                let Ok(result) =
+                let Ok(result) = tokio::task::spawn_blocking(move || {
-                    tokio::task::spawn_blocking(move || Publisher::publish_crate(data, client))
+                    Publisher::publish_crate(data, client, index)
-                        .await
+                })
+                .await
                 else {
                     return response_builder::error_response(
                         hyper::StatusCode::INTERNAL_SERVER_ERROR,
@@ -218,7 +221,7 @@ impl CargoRegistryService {
     }
 
     async fn handler(
-        index: sparse_index::RegistryIndex,
+        index: Arc<sparse_index::RegistryIndex>,
         request: hyper::Request<hyper::Body>,
         client: Arc<Client>,
     ) -> Result<hyper::Response<hyper::Body>, Error> {
@@ -257,7 +260,7 @@ impl CargoRegistryService {
                             "Invalid length of the request.",
                         )
                     } else {
-                        Self::handle_publish_request(request, client.clone()).await
+                        Self::handle_publish_request(request, client.clone(), index.clone()).await
                     }
                 }
                 "unyank" => Self::handle_unyank_request(path, &request),
@@ -297,7 +300,10 @@ async fn main() {
     let client = Arc::new(Client::new().expect("Failed to get RPC Client instance"));
 
     let bind_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), client.port);
-    let index = sparse_index::RegistryIndex::new("/index", &client.server_url);
+    let index = Arc::new(sparse_index::RegistryIndex::new(
+        "/index",
+        &client.server_url,
+    ));
 
     let registry_service = make_service_fn(move |_| {
         let client_inner = client.clone();
@@ -310,7 +316,7 @@ async fn main() {
     });
 
     let server = Server::bind(&bind_addr).serve(registry_service);
-    info!("Server running on on http://{}", bind_addr);
+    info!("Server running on http://{}", bind_addr);
 
     let _ = server.await;
 }

cargo-registry/src/publisher.rs

@@ -1,10 +1,14 @@
 use {
-    crate::client::{Client, ClientConfig},
+    crate::{
+        client::{Client, ClientConfig},
+        sparse_index::{IndexEntry, RegistryIndex},
+    },
     flate2::read::GzDecoder,
     hyper::body::Bytes,
     log::*,
     serde::{Deserialize, Serialize},
     serde_json::from_slice,
+    sha2::{Digest, Sha256},
     solana_cli::program_v4::{process_deploy_program, read_and_verify_elf},
     solana_sdk::{
         signature::{Keypair, Signer},
@@ -22,11 +26,11 @@ use {
     tempfile::{tempdir, TempDir},
 };
 
-pub type Error = Box<dyn std::error::Error + Send + Sync + 'static>;
+pub(crate) type Error = Box<dyn std::error::Error + Send + Sync + 'static>;
 
 #[derive(Debug, Deserialize, Serialize)]
 #[serde(rename_all = "lowercase")]
-enum DependencyType {
+pub(crate) enum DependencyType {
     Dev,
     Build,
     Normal,
@@ -34,39 +38,39 @@ enum DependencyType {
 
 #[allow(dead_code)]
 #[derive(Debug, Deserialize)]
-struct Dependency {
+pub(crate) struct Dependency {
-    name: String,
+    pub name: String,
-    version_req: String,
+    pub version_req: String,
-    features: Vec<String>,
+    pub features: Vec<String>,
-    optional: bool,
+    pub optional: bool,
-    default_features: bool,
+    pub default_features: bool,
-    target: Option<String>,
+    pub target: Option<String>,
-    kind: DependencyType,
+    pub kind: DependencyType,
-    registry: Option<String>,
+    pub registry: Option<String>,
-    explicit_name_in_toml: Option<String>,
+    pub explicit_name_in_toml: Option<String>,
 }
 
 #[derive(Debug, Deserialize)]
 #[allow(unused)]
-struct PackageMetaData {
+pub(crate) struct PackageMetaData {
-    name: String,
+    pub name: String,
-    vers: String,
+    pub vers: String,
-    deps: Vec<Dependency>,
+    pub deps: Vec<Dependency>,
-    features: BTreeMap<String, Vec<String>>,
+    pub features: BTreeMap<String, Vec<String>>,
-    authors: Vec<String>,
+    pub authors: Vec<String>,
-    description: Option<String>,
+    pub description: Option<String>,
-    documentation: Option<String>,
+    pub documentation: Option<String>,
-    homepage: Option<String>,
+    pub homepage: Option<String>,
-    readme: Option<String>,
+    pub readme: Option<String>,
-    readme_file: Option<String>,
+    pub readme_file: Option<String>,
-    keywords: Vec<String>,
+    pub keywords: Vec<String>,
-    categories: Vec<String>,
+    pub categories: Vec<String>,
-    license: Option<String>,
+    pub license: Option<String>,
-    license_file: Option<String>,
+    pub license_file: Option<String>,
-    repository: Option<String>,
+    pub repository: Option<String>,
-    badges: BTreeMap<String, BTreeMap<String, String>>,
+    pub badges: BTreeMap<String, BTreeMap<String, String>>,
-    links: Option<String>,
+    pub links: Option<String>,
-    rust_version: Option<String>,
+    pub rust_version: Option<String>,
 }
 
 impl PackageMetaData {
@@ -86,7 +90,7 @@ impl PackageMetaData {
     }
 }
 
-pub struct Publisher {}
+pub(crate) struct Publisher {}
 
 impl Publisher {
     fn make_path<P: AsRef<Path>>(tempdir: &TempDir, meta: &PackageMetaData, append: P) -> PathBuf {
@@ -107,12 +111,17 @@ impl Publisher {
         Ok(library_name.to_string())
     }
 
-    pub(crate) fn publish_crate(bytes: Bytes, client: Arc<Client>) -> Result<(), Error> {
+    pub(crate) fn publish_crate(
+        bytes: Bytes,
+        client: Arc<Client>,
+        index: Arc<RegistryIndex>,
+    ) -> Result<(), Error> {
         let (meta_data, offset) = PackageMetaData::new(&bytes)?;
 
         let (_crate_file_length, length_size) =
             PackageMetaData::read_u32_length(&bytes.slice(offset..))?;
         let crate_bytes = bytes.slice(offset.saturating_add(length_size)..);
+        let crate_cksum = format!("{:x}", Sha256::digest(&crate_bytes));
 
         let decoder = GzDecoder::new(crate_bytes.as_ref());
         let mut archive = Archive::new(decoder);
@@ -154,6 +163,10 @@ impl Publisher {
             format!("Failed to deploy the program: {}", e)
         })?;
 
+        let mut entry: IndexEntry = meta_data.into();
+        entry.cksum = crate_cksum;
+        index.insert_entry(entry)?;
+
         info!("Successfully deployed the program");
         Ok(())
     }

cargo-registry/src/sparse_index.rs

@@ -1,7 +1,11 @@
 use {
-    crate::response_builder,
+    crate::{
+        publisher::{Dependency, Error, PackageMetaData},
+        response_builder,
+    },
     log::info,
     serde::{Deserialize, Serialize},
+    std::{collections::BTreeMap, sync::RwLock},
 };
 
 #[derive(Debug, Default, Deserialize, Serialize)]
@@ -10,14 +14,68 @@ struct RegistryConfig {
     api: Option<String>,
 }
 
-#[derive(Clone)]
+pub(crate) struct RegistryIndex {
-pub struct RegistryIndex {
     pub(crate) index_root: String,
     config: String,
+    index: RwLock<BTreeMap<String, IndexEntry>>,
+}
+
+#[derive(Serialize)]
+pub(crate) struct IndexEntryDep {
+    pub name: String,
+    pub req: String,
+    pub features: Vec<String>,
+    pub optional: bool,
+    pub default_features: bool,
+    pub target: Option<String>,
+    pub kind: String,
+    pub registry: Option<String>,
+    pub package: Option<String>,
+}
+
+impl From<Dependency> for IndexEntryDep {
+    fn from(v: Dependency) -> Self {
+        IndexEntryDep {
+            name: v.name,
+            req: v.version_req,
+            features: v.features,
+            optional: v.optional,
+            default_features: v.default_features,
+            target: v.target,
+            kind: serde_json::to_string(&v.kind).expect("Failed to stringify dep kind"),
+            registry: v.registry,
+            package: None,
+        }
+    }
+}
+
+#[derive(Serialize)]
+pub(crate) struct IndexEntry {
+    pub name: String,
+    pub vers: String,
+    pub deps: Vec<IndexEntryDep>,
+    pub cksum: String,
+    pub features: BTreeMap<String, Vec<String>>,
+    pub yanked: bool,
+    pub links: Option<String>,
+}
+
+impl From<PackageMetaData> for IndexEntry {
+    fn from(v: PackageMetaData) -> Self {
+        IndexEntry {
+            name: v.name,
+            vers: v.vers,
+            deps: v.deps.into_iter().map(|v| v.into()).collect(),
+            cksum: String::new(),
+            features: v.features,
+            yanked: false,
+            links: v.links,
+        }
+    }
 }
 
 impl RegistryIndex {
-    pub fn new(root: &str, server_url: &str) -> Self {
+    pub(crate) fn new(root: &str, server_url: &str) -> Self {
         let registry_config = RegistryConfig {
             dl: format!("{}/api/v1/crates", server_url),
             api: Some(server_url.to_string()),
@@ -29,10 +87,14 @@ impl RegistryIndex {
         Self {
             index_root: root.to_string(),
             config,
+            index: RwLock::new(BTreeMap::new()),
         }
     }
 
-    pub fn handler(&self, request: hyper::Request<hyper::Body>) -> hyper::Response<hyper::Body> {
+    pub(crate) fn handler(
+        &self,
+        request: hyper::Request<hyper::Body>,
+    ) -> hyper::Response<hyper::Body> {
         let path = request.uri().path();
         let expected_root = self.index_root.as_str();
         if !path.starts_with(expected_root) {
@@ -53,7 +115,17 @@ impl RegistryIndex {
             return response_builder::success_response_str(&self.config);
         }
 
-        Self::handle_crate_lookup_request(path)
+        self.handle_crate_lookup_request(path)
+    }
+
+    pub(crate) fn insert_entry(&self, entry: IndexEntry) -> Result<(), Error> {
+        let mut write_index = self
+            .index
+            .write()
+            .map_err(|e| format!("Failed to lock the index for writing: {}", e))?;
+        info!("Inserting {}-{} in registry index", entry.name, entry.vers);
+        write_index.insert(entry.name.clone(), entry);
+        Ok(())
     }
 
     fn get_crate_name_from_path(path: &str) -> Option<&str> {
@@ -78,7 +150,7 @@ impl RegistryIndex {
         .then_some(crate_name)
     }
 
-    fn handle_crate_lookup_request(path: &str) -> hyper::Response<hyper::Body> {
+    fn handle_crate_lookup_request(&self, path: &str) -> hyper::Response<hyper::Body> {
         let Some(crate_name) = Self::get_crate_name_from_path(path) else {
             return response_builder::error_response(
                 hyper::StatusCode::BAD_REQUEST,
@@ -86,10 +158,31 @@ impl RegistryIndex {
             );
         };
 
-        // Fetch the index information for the crate
+        info!("Looking up index for {:?}", crate_name);
-        info!("Received a request to fetch {:?}", crate_name);
 
-        response_builder::success_response()
+        let Ok(read_index) = self.index.read() else {
+            return response_builder::error_response(
+                hyper::StatusCode::INTERNAL_SERVER_ERROR,
+                "Internal error. Failed to lock the index for reading",
+            );
+        };
+
+        let Some(entry) = read_index.get(crate_name) else {
+            // The index currently doesn't contain the program entry.
+            // Fetch the program information from the network using RPC client.
+            // In the meanwhile, return empty success response, so that the registry
+            // client continues to poll us for the index information.
+            return response_builder::success_response();
+        };
+
+        let Ok(response) = serde_json::to_string(entry) else {
+            return response_builder::error_response(
+                hyper::StatusCode::INTERNAL_SERVER_ERROR,
+                "Internal error. index entry is corrupted",
+            );
+        };
+
+        response_builder::success_response_str(response.as_str())
     }
 }