From 6e32ef2f8c4f84dd7123b5b96780e68dcdfa2ada Mon Sep 17 00:00:00 2001 From: Krzysztof Lis Date: Fri, 22 Dec 2023 01:10:41 +0100 Subject: [PATCH 1/2] feat(gateway-client): add api key support --- Cargo.lock | 2 + Cargo.toml | 1 + crates/ethereum/Cargo.toml | 2 +- crates/gateway-client/Cargo.toml | 2 + crates/gateway-client/src/builder.rs | 160 ++++++++++++++++-- crates/gateway-client/src/lib.rs | 17 +- crates/gateway-types/Cargo.toml | 2 +- .../pathfinder/src/bin/pathfinder/config.rs | 10 ++ crates/pathfinder/src/bin/pathfinder/main.rs | 40 +++-- 9 files changed, 205 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e90e8df3e..06b339d70d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7750,9 +7750,11 @@ dependencies = [ "async-trait", "base64 0.13.1", "bytes", + "fake", "flate2", "futures", "http", + "httpmock", "lazy_static", "metrics", "mockall", diff --git a/Cargo.toml b/Cargo.toml index 1d8a9fdf3b..c300e5d777 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ flate2 = "1.0.27" futures = { version = "0.3", default-features = false, features = ["std"] } hex = "0.4.3" http = "0.2.9" +httpmock = "0.7.0-rc.1" lazy_static = "1.4.0" metrics = "0.20.1" num-bigint = { version = "0.4.4", features = ["serde"] } diff --git a/crates/ethereum/Cargo.toml b/crates/ethereum/Cargo.toml index e13650b104..b98832e9ac 100644 --- a/crates/ethereum/Cargo.toml +++ b/crates/ethereum/Cargo.toml @@ -23,5 +23,5 @@ tokio = { workspace = true } tracing = { workspace = true } [dev-dependencies] -httpmock = "0.7.0-rc.1" +httpmock = { workspace = true } tokio = { workspace = true, features = ["macros"] } diff --git a/crates/gateway-client/Cargo.toml b/crates/gateway-client/Cargo.toml index 2ec8da3148..4e84e74fe8 100644 --- a/crates/gateway-client/Cargo.toml +++ b/crates/gateway-client/Cargo.toml @@ -32,7 +32,9 @@ warp = { version = "0.3.5" } [dev-dependencies] assert_matches = { workspace = true } base64 = { workspace = true } +fake = { workspace = true } flate2 = { workspace = true } +httpmock = { workspace = true } lazy_static = { workspace = true } pathfinder-crypto = { path = "../crypto" } pretty_assertions_sorted = { workspace = true } diff --git a/crates/gateway-client/src/builder.rs b/crates/gateway-client/src/builder.rs index 07161e78cc..64c9ee3c27 100644 --- a/crates/gateway-client/src/builder.rs +++ b/crates/gateway-client/src/builder.rs @@ -13,10 +13,13 @@ use crate::metrics::{with_metrics, BlockTag, RequestMetadata}; use pathfinder_common::{BlockId, ClassHash, TransactionHash}; use starknet_gateway_types::error::SequencerError; +const X_THROTTLING_BYPASS: &str = "X-Throttling-Bypass"; + /// A Sequencer Request builder. pub struct Request<'a, S: RequestState> { state: S, url: reqwest::Url, + api_key: Option, client: &'a reqwest::Client, } @@ -64,10 +67,15 @@ pub mod stage { impl<'a> Request<'a, stage::Init> { /// Initialize a [Request] builder. - pub fn builder(client: &'a reqwest::Client, url: reqwest::Url) -> Request<'a, stage::Method> { + pub fn builder( + client: &'a reqwest::Client, + url: reqwest::Url, + api_key: Option, + ) -> Request<'a, stage::Method> { Request { url, client, + api_key, state: stage::Method, } } @@ -144,6 +152,7 @@ impl<'a> Request<'a, stage::Method> { Request { url: self.url, client: self.client, + api_key: self.api_key, state: stage::Params { meta: RequestMetadata::new(method), }, @@ -201,6 +210,7 @@ impl<'a> Request<'a, stage::Params> { Request { url: self.url, client: self.client, + api_key: self.api_key, state: stage::Final { meta: self.state.meta, retry, @@ -217,24 +227,31 @@ impl<'a> Request<'a, stage::Final> { { async fn send_request( url: reqwest::Url, + api_key: Option, client: &reqwest::Client, meta: RequestMetadata, ) -> Result { with_metrics(meta, async move { tracing::trace!(%url, "Fetching data from feeder gateway"); - let response = client.get(url).send().await?; + let request = client.get(url); + let request = match api_key { + Some(api_key) => request.header(X_THROTTLING_BYPASS, api_key), + None => request, + }; + let response = request.send().await?; parse::(response).await }) .await } match self.state.retry { - false => send_request(self.url, self.client, self.state.meta).await, + false => send_request(self.url, self.api_key, self.client, self.state.meta).await, true => { retry0( || async { - let clone_url = self.url.clone(); - send_request(clone_url, self.client, self.state.meta).await + let url = self.url.clone(); + let api_key = self.api_key.clone(); + send_request(url, api_key, self.client, self.state.meta).await }, retry_condition, ) @@ -247,12 +264,18 @@ impl<'a> Request<'a, stage::Final> { pub async fn get_as_bytes(self) -> Result { async fn get_as_bytes_inner( url: reqwest::Url, + api_key: Option, client: &reqwest::Client, meta: RequestMetadata, ) -> Result { with_metrics(meta, async { tracing::trace!(%url, "Fetching binary data from feeder gateway"); - let response = client.get(url).send().await?; + let request = client.get(url); + let request = match api_key { + Some(api_key) => request.header(X_THROTTLING_BYPASS, api_key), + None => request, + }; + let response = request.send().await?; let response = parse_raw(response).await?; let bytes = response.bytes().await?; Ok(bytes) @@ -261,12 +284,13 @@ impl<'a> Request<'a, stage::Final> { } match self.state.retry { - false => get_as_bytes_inner(self.url, self.client, self.state.meta).await, + false => get_as_bytes_inner(self.url, self.api_key, self.client, self.state.meta).await, true => { retry0( || async { - let clone_url = self.url.clone(); - get_as_bytes_inner(clone_url, self.client, self.state.meta).await + let url = self.url.clone(); + let api_key = self.api_key.clone(); + get_as_bytes_inner(url, api_key, self.client, self.state.meta).await }, retry_condition, ) @@ -284,6 +308,7 @@ impl<'a> Request<'a, stage::Final> { { async fn post_with_json_inner( url: reqwest::Url, + api_key: Option, client: &reqwest::Client, meta: RequestMetadata, json: &J, @@ -293,20 +318,29 @@ impl<'a> Request<'a, stage::Final> { J: serde::Serialize + ?Sized, { with_metrics(meta, async { - let response = client.post(url).json(json).send().await?; + let request = client.post(url); + let request = match api_key { + Some(api_key) => request.header(X_THROTTLING_BYPASS, api_key), + None => request, + }; + let response = request.json(json).send().await?; parse::(response).await }) .await } match self.state.retry { - false => post_with_json_inner(self.url, self.client, self.state.meta, json).await, + false => { + post_with_json_inner(self.url, self.api_key, self.client, self.state.meta, json) + .await + } true => { retry0( || async { tracing::trace!(url=%self.url, "Posting data to gateway"); - let clone_url = self.url.clone(); - post_with_json_inner(clone_url, self.client, self.state.meta, json).await + let url = self.url.clone(); + let api_key = self.api_key.clone(); + post_with_json_inner(url, api_key, self.client, self.state.meta, json).await }, retry_condition, ) @@ -596,4 +630,104 @@ mod tests { ); } } + + mod api_key_is_set_when_configured { + use crate::Client; + use fake::{Fake, Faker}; + use httpmock::{prelude::*, Mock}; + use serde_json::json; + + async fn setup_with_fake_api_key(server: &MockServer) -> (Mock<'_>, Client) { + let api_key = Faker.fake::(); + + let mock = server.mock(|when, then| { + when.any_request().header("X-Throttling-Bypass", &api_key); + then.status(200).json_body(json!({})); + }); + + let client = Client::with_base_url(server.base_url().parse().unwrap()) + .unwrap() + .with_api_key(Some(api_key.clone())); + + (mock, client) + } + + #[tokio::test] + async fn get() -> anyhow::Result<()> { + let server = MockServer::start_async().await; + let (mock, client) = setup_with_fake_api_key(&server).await; + + let _: serde_json::Value = client + .clone() + .gateway_request() + .with_method("") + .with_retry(false) + .get() + .await?; + + let _: serde_json::Value = client + .clone() + .feeder_gateway_request() + .with_method("") + .with_retry(false) + .get() + .await?; + + mock.assert_hits(2); + + Ok(()) + } + + #[tokio::test] + async fn get_as_bytes() -> anyhow::Result<()> { + let server = MockServer::start_async().await; + let (mock, client) = setup_with_fake_api_key(&server).await; + + let _: bytes::Bytes = client + .clone() + .gateway_request() + .with_method("") + .with_retry(false) + .get_as_bytes() + .await?; + + let _: bytes::Bytes = client + .clone() + .feeder_gateway_request() + .with_method("") + .with_retry(false) + .get_as_bytes() + .await?; + + mock.assert_hits(2); + + Ok(()) + } + + #[tokio::test] + async fn post_with_json() -> anyhow::Result<()> { + let server = MockServer::start_async().await; + let (mock, client) = setup_with_fake_api_key(&server).await; + + let _: serde_json::Value = client + .clone() + .gateway_request() + .with_method("") + .with_retry(false) + .post_with_json(&json!({})) + .await?; + + let _: serde_json::Value = client + .clone() + .feeder_gateway_request() + .with_method("") + .with_retry(false) + .post_with_json(&json!({})) + .await?; + + mock.assert_hits(2); + + Ok(()) + } + } } diff --git a/crates/gateway-client/src/lib.rs b/crates/gateway-client/src/lib.rs index 59ccd3f22a..c8a2a87890 100644 --- a/crates/gateway-client/src/lib.rs +++ b/crates/gateway-client/src/lib.rs @@ -240,6 +240,8 @@ pub struct Client { /// Whether __read only__ requests should be retried, defaults to __true__ for production. /// Use [disable_retry_for_tests](Client::disable_retry_for_tests) to disable retry logic for all __read only__ requests when testing. retry: bool, + /// Api key added to each request as a value for 'X-Throttling-Bypass' header. + api_key: Option, } impl Client { @@ -289,9 +291,16 @@ impl Client { gateway, feeder_gateway, retry: true, + api_key: None, }) } + /// Sets the api key to be used for each request as a value for 'X-Throttling-Bypass' header. + pub fn with_api_key(mut self, api_key: Option) -> Self { + self.api_key = api_key; + self + } + /// Use this method to disable retry logic for all __non write__ requests when testing. pub fn disable_retry_for_tests(self) -> Self { Self { @@ -301,11 +310,15 @@ impl Client { } fn gateway_request(&self) -> builder::Request<'_, builder::stage::Method> { - builder::Request::builder(&self.inner, self.gateway.clone()) + builder::Request::builder(&self.inner, self.gateway.clone(), self.api_key.clone()) } fn feeder_gateway_request(&self) -> builder::Request<'_, builder::stage::Method> { - builder::Request::builder(&self.inner, self.feeder_gateway.clone()) + builder::Request::builder( + &self.inner, + self.feeder_gateway.clone(), + self.api_key.clone(), + ) } async fn block_with_retry_behaviour( diff --git a/crates/gateway-types/Cargo.toml b/crates/gateway-types/Cargo.toml index 895cd6aa50..5501d172fe 100644 --- a/crates/gateway-types/Cargo.toml +++ b/crates/gateway-types/Cargo.toml @@ -9,7 +9,7 @@ rust-version = { workspace = true } [dependencies] anyhow = { workspace = true } -fake = { workspace = true } +fake = { workspace = true, features = ["serde_json"] } lazy_static = { workspace = true } pathfinder-common = { path = "../common" } pathfinder-crypto = { path = "../crypto" } diff --git a/crates/pathfinder/src/bin/pathfinder/config.rs b/crates/pathfinder/src/bin/pathfinder/config.rs index 202162822f..3651fa21b1 100644 --- a/crates/pathfinder/src/bin/pathfinder/config.rs +++ b/crates/pathfinder/src/bin/pathfinder/config.rs @@ -197,6 +197,14 @@ This should only be enabled for debugging purposes as it adds substantial proces action=ArgAction::Set )] is_rpc_enabled: bool, + + #[arg( + long = "gateway-api-key", + value_name = "API_KEY", + long_help = "Specify an API key for both the Starknet feeder gateway and gateway.", + env = "PATHFINDER_GATEWAY_API_KEY" + )] + gateway_api_key: Option, } #[derive(clap::ValueEnum, Debug, Clone, Copy, PartialEq)] @@ -438,6 +446,7 @@ pub struct Config { pub rpc_batch_concurrency_limit: NonZeroUsize, pub is_sync_enabled: bool, pub is_rpc_enabled: bool, + pub gateway_api_key: Option, } pub struct Ethereum { @@ -613,6 +622,7 @@ impl Config { rpc_batch_concurrency_limit: cli.rpc_batch_concurrency_limit, is_sync_enabled: cli.is_sync_enabled, is_rpc_enabled: cli.is_rpc_enabled, + gateway_api_key: cli.gateway_api_key, } } } diff --git a/crates/pathfinder/src/bin/pathfinder/main.rs b/crates/pathfinder/src/bin/pathfinder/main.rs index dd76a3caca..fba926a1c1 100644 --- a/crates/pathfinder/src/bin/pathfinder/main.rs +++ b/crates/pathfinder/src/bin/pathfinder/main.rs @@ -94,10 +94,13 @@ async fn async_main() -> anyhow::Result<()> { .context("Starting monitoring task")?; } - let pathfinder_context = - PathfinderContext::configure_and_proxy_check(network, config.data_directory) - .await - .context("Configuring pathfinder")?; + let pathfinder_context = PathfinderContext::configure_and_proxy_check( + network, + config.data_directory, + config.gateway_api_key, + ) + .await + .context("Configuring pathfinder")?; verify_networks(pathfinder_context.network, ethereum.chain)?; @@ -489,40 +492,41 @@ mod pathfinder_context { pub async fn configure_and_proxy_check( cfg: NetworkConfig, data_directory: PathBuf, + api_key: Option, ) -> anyhow::Result { let context = match cfg { NetworkConfig::Mainnet => Self { network: Chain::Mainnet, network_id: ChainId::MAINNET, - gateway: GatewayClient::mainnet(), + gateway: GatewayClient::mainnet().with_api_key(api_key), database: data_directory.join("mainnet.sqlite"), l1_core_address: H160::from(core_addr::MAINNET), }, NetworkConfig::GoerliTestnet => Self { network: Chain::GoerliTestnet, network_id: ChainId::GOERLI_TESTNET, - gateway: GatewayClient::goerli_testnet(), + gateway: GatewayClient::goerli_testnet().with_api_key(api_key), database: data_directory.join("goerli.sqlite"), l1_core_address: H160::from(core_addr::GOERLI_TESTNET), }, NetworkConfig::GoerliIntegration => Self { network: Chain::GoerliIntegration, network_id: ChainId::GOERLI_INTEGRATION, - gateway: GatewayClient::goerli_integration(), + gateway: GatewayClient::goerli_integration().with_api_key(api_key), database: data_directory.join("integration.sqlite"), l1_core_address: H160::from(core_addr::GOERLI_INTEGRATION), }, NetworkConfig::SepoliaTestnet => Self { network: Chain::SepoliaTestnet, network_id: ChainId::SEPOLIA_TESTNET, - gateway: GatewayClient::sepolia_testnet(), + gateway: GatewayClient::sepolia_testnet().with_api_key(api_key), database: data_directory.join("testnet-sepolia.sqlite"), l1_core_address: H160::from(core_addr::SEPOLIA_TESTNET), }, NetworkConfig::SepoliaIntegration => Self { network: Chain::SepoliaIntegration, network_id: ChainId::SEPOLIA_INTEGRATION, - gateway: GatewayClient::sepolia_integration(), + gateway: GatewayClient::sepolia_integration().with_api_key(api_key), database: data_directory.join("integration-sepolia.sqlite"), l1_core_address: H160::from(core_addr::SEPOLIA_INTEGRATION), }, @@ -530,9 +534,15 @@ mod pathfinder_context { gateway, feeder_gateway, chain_id, - } => Self::configure_custom(gateway, feeder_gateway, chain_id, data_directory) - .await - .context("Configuring custom network")?, + } => Self::configure_custom( + gateway, + feeder_gateway, + chain_id, + data_directory, + api_key, + ) + .await + .context("Configuring custom network")?, }; Ok(context) @@ -546,12 +556,14 @@ mod pathfinder_context { feeder: Url, chain_id: String, data_directory: PathBuf, + api_key: Option, ) -> anyhow::Result { use pathfinder_crypto::Felt; use starknet_gateway_client::GatewayApi; - let gateway = - GatewayClient::with_urls(gateway, feeder).context("Creating gateway client")?; + let gateway = GatewayClient::with_urls(gateway, feeder) + .context("Creating gateway client")? + .with_api_key(api_key); let network_id = ChainId(Felt::from_be_slice(chain_id.as_bytes()).context("Parsing chain ID")?); From 8c15ba6172cd044dd5a6ba49a3086a8bb48a5da2 Mon Sep 17 00:00:00 2001 From: Krzysztof Lis Date: Fri, 22 Dec 2023 17:07:33 +0100 Subject: [PATCH 2/2] doc: update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31b942ba0c..d319120005 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added + +- `gateway-api-key API_KEY` configuration option. If enabled, each time a request is sent to the Starknet gateway or the feeder gateway a `X-Throttling-Bypass: API_KEY` header will be set. + ## [0.10.3-rc0] - 2023-12-14 ### Changed