Add Database CLI tool (#34)

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = [".", "auth", "auth-sled"]
+members = [".", "auth", "auth-sled", "db-cli"]
 
 [package]
 name = "webb-faucet"

auth-sled/src/lib.rs

@@ -1,8 +1,9 @@
 use chrono::{DateTime, Utc};
 use std::convert::TryFrom;
-use webb_auth::{model::ClaimsData, AuthDb, UserInfo};
+use webb_auth::AuthDb;
 use webb_proposals::TypedChainId;
 
+pub use webb_auth::model::*;
 /// SledStore is a store that stores the history of events in  a [Sled](https://sled.rs)-based database.
 #[derive(Clone)]
 pub struct SledAuthDb {
@@ -20,6 +21,19 @@ impl SledAuthDb {
         })
     }
 
+    pub fn user_info_tree(&self) -> Result<sled::Tree, Error> {
+        self.db.open_tree("users").map_err(Into::into)
+    }
+
+    pub fn claims_tree(
+        &self,
+        chain_id: TypedChainId,
+    ) -> Result<sled::Tree, Error> {
+        self.db
+            .open_tree(format!("claims-{}", chain_id.chain_id()))
+            .map_err(Into::into)
+    }
+
     /// Open a new SledStore in a temporary directory.
     #[cfg(test)]
     pub fn open_for_tests() -> Result<Self, Error> {
@@ -39,7 +53,7 @@ impl AuthDb for SledAuthDb {
         value: &UserInfo,
     ) -> Result<(), Self::Error> {
         let id = u64_to_i64(id)?;
-        let user_info_tree = self.db.open_tree("users")?;
+        let user_info_tree = self.user_info_tree()?;
         let user_info_bytes = serde_json::to_vec(value)?;
         user_info_tree.insert(id.to_be_bytes(), user_info_bytes)?;
         Ok(())
@@ -50,7 +64,7 @@ impl AuthDb for SledAuthDb {
         id: u64,
     ) -> Result<Option<UserInfo>, Self::Error> {
         let id = u64_to_i64(id)?;
-        let user_info_tree = self.db.open_tree("users")?;
+        let user_info_tree = self.user_info_tree()?;
         user_info_tree
             .get(id.to_be_bytes())
             .map_err(Into::into)
@@ -67,9 +81,7 @@ impl AuthDb for SledAuthDb {
         claim: ClaimsData,
     ) -> Result<DateTime<Utc>, Self::Error> {
         let id = u64_to_i64(id)?;
-        let last_claim_tree = self
-            .db
-            .open_tree(format!("claims-{}", typed_chain_id.chain_id()))?;
+        let last_claim_tree = self.claims_tree(typed_chain_id)?;
         let claims_data_bytes = serde_json::to_vec(&claim)?;
         last_claim_tree.insert(id.to_be_bytes(), claims_data_bytes)?;
         Ok(claim.last_claimed_date)
@@ -81,9 +93,7 @@ impl AuthDb for SledAuthDb {
         typed_chain_id: TypedChainId,
     ) -> Result<Option<ClaimsData>, Self::Error> {
         let id = u64_to_i64(id)?;
-        let last_claim_tree = self
-            .db
-            .open_tree(format!("claims-{}", typed_chain_id.chain_id()))?;
+        let last_claim_tree = self.claims_tree(typed_chain_id)?;
         last_claim_tree
             .get(id.to_be_bytes())
             .map_err(Into::into)
@@ -94,16 +104,22 @@ impl AuthDb for SledAuthDb {
     }
 }
 
-fn u64_to_i64(value: u64) -> Result<i64, Error> {
-    i64::try_from(value).map_err(|_| Error::InvalidId(value))
+pub fn u64_to_i64(value: u64) -> Result<i64, Error> {
+    i64::try_from(value).map_err(|_| Error::InvalidU65Id(value))
+}
+
+pub fn i64_to_u64(value: i64) -> Result<u64, Error> {
+    u64::try_from(value).map_err(|_| Error::InvalidI65Id(value))
 }
 
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
     #[error("Sled error: {0}")]
     Sled(#[from] sled::Error),
     #[error("Invalid ID: {0}")]
-    InvalidId(u64),
+    InvalidU65Id(u64),
+    #[error("Invalid ID: {0}")]
+    InvalidI65Id(i64),
     #[error("Invalid Serialization: {0}")]
     Serde(#[from] serde_json::Error),
 }

auth/Cargo.toml

@@ -10,4 +10,6 @@ async-trait = "0.1"
 chrono = { version = "0.4", features = ["serde"] }
 serde = "1.0"
 thiserror = "1.0"
+ethers-core = "2.0.10"
+sp-core = "27.0"
 webb-proposals = { git = "https://github.com/webb-tools/webb-rs", rev="a960eaf", features = ["scale"] }

auth/src/model.rs

@@ -1,4 +1,6 @@
 use chrono::{DateTime, Utc};
+use ethers_core::types::Address;
+use sp_core::crypto::{AccountId32, Ss58Codec};
 
 #[derive(
     Copy,
@@ -55,6 +57,34 @@ impl UniversalWalletAddress {
     pub fn is_substrate(&self) -> bool {
         matches!(self, Self::Substrate(..))
     }
+
+    pub fn as_ethereum(&self) -> Option<Address> {
+        if let Self::Ethereum(v) = self {
+            Some(Address::from(*v))
+        } else {
+            None
+        }
+    }
+
+    pub fn as_substrate(&self) -> Option<AccountId32> {
+        if let Self::Substrate(v) = self {
+            Some(AccountId32::from(*v))
+        } else {
+            None
+        }
+    }
+}
+
+impl core::fmt::Display for UniversalWalletAddress {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Unknown => write!(f, "Unknown"),
+            Self::Ethereum(v) => write!(f, "{:?}", Address::from(*v)),
+            Self::Substrate(v) => {
+                write!(f, "{}", AccountId32::from(*v).to_ss58check())
+            }
+        }
+    }
 }
 
 #[derive(

db-cli/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "db-cli"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["parking_lot", "env-filter"] }
+color-eyre = "0.6"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+argh = "0.1"
+webb-auth-sled = { path = "../auth-sled" }
+webb-proposals = { git = "https://github.com/webb-tools/webb-rs", rev="a960eaf", features = ["scale"] }

db-cli/README.md

@@ -0,0 +1,30 @@
+## How to use the CLI
+
+`db-cli` is a command line interface for the database. It reads the database and output all the accounts in a JSON format.
+
+**Usage:**
+```bash
+Usage: db-cli -d <db> [-v <verbosity>] [-e <evm-output>] [-s <substrate-output>]
+
+Webb Faucet Database CLI
+
+Options:
+  -d, --db          sled database path
+  -v, --verbosity   control verbosity level
+  -e, --evm-output  output file for evm addresses
+  -s, --substrate-output
+                    output file for substrate addresses
+  --help            display usage information
+```
+
+## Examples
+
+1. Output all the accounts in the database to stderr
+```bash
+./db-cli -d ./faucet
+```
+2. Output all the accounts in the database to json files
+
+```bash
+./db-cli -d ./faucet -v 2 -e evm.json -s substrate.json
+```

db-cli/src/main.rs

@@ -0,0 +1,114 @@
+use std::collections::HashSet;
+
+use argh::FromArgs;
+use color_eyre::eyre::Result;
+use webb_auth_sled::{ClaimsData, SledAuthDb};
+use webb_proposals::TypedChainId;
+
+/// Webb Faucet Database CLI
+#[derive(Debug, Clone, FromArgs)]
+struct Args {
+    /// sled database path
+    #[argh(option, short = 'd')]
+    db: std::path::PathBuf,
+
+    /// control verbosity level
+    #[argh(option, short = 'v', default = "0")]
+    verbosity: u8,
+    /// output file for evm addresses
+    #[argh(option, short = 'e')]
+    evm_output: Option<std::path::PathBuf>,
+    /// output file for substrate addresses
+    #[argh(option, short = 's')]
+    substrate_output: Option<std::path::PathBuf>,
+}
+
+fn main() -> Result<()> {
+    color_eyre::install()?;
+    let args: Args = argh::from_env();
+    setup_logger(args.verbosity, "db_cli")?;
+    tracing::info!("opening db at {:?}", args.db);
+    let db = SledAuthDb::open(args.db)?;
+    let chains = [
+        // Tangle
+        TypedChainId::Substrate(1081),
+        // Tangle Local
+        TypedChainId::Substrate(1082),
+        // Athena
+        TypedChainId::Evm(3884533461),
+        // Demeter
+        TypedChainId::Evm(3884533463),
+        // Hermes
+        TypedChainId::Evm(3884533462),
+        // Tangle EVM Testnet
+        TypedChainId::Evm(4006),
+    ];
+
+    let mut accounts = HashSet::new();
+
+    for chain in chains.iter() {
+        tracing::debug!(
+            chain = %chain.chain_id(),
+            "processing chain claims",
+        );
+
+        let chain_accounts = db.claims_tree(*chain)?.iter().flat_map(|kv| {
+            kv.ok()
+                .and_then(|(_, v)| {
+                    serde_json::from_slice::<ClaimsData>(&v).ok()
+                })
+                .map(|c| c.address)
+        });
+        accounts.extend(chain_accounts);
+        tracing::debug!("Total accounts (so far): {}", accounts.len());
+    }
+    let evm_accounts = accounts
+        .iter()
+        .filter_map(|a| a.as_ethereum().map(|v| format!("{:?}", v)))
+        .collect::<Vec<_>>();
+    let substrate_accounts = accounts
+        .iter()
+        .filter_map(|a| a.as_substrate().map(|v| v.to_string()))
+        .collect::<Vec<_>>();
+    if let Some(output) = args.evm_output {
+        std::fs::write(output, serde_json::to_string_pretty(&evm_accounts)?)?;
+    } else {
+        eprintln!("{}", serde_json::to_string_pretty(&evm_accounts)?);
+    }
+    if let Some(output) = args.substrate_output {
+        std::fs::write(
+            output,
+            serde_json::to_string_pretty(&substrate_accounts)?,
+        )?;
+    } else {
+        eprintln!("{}", serde_json::to_string_pretty(&substrate_accounts)?);
+    }
+    Ok(())
+}
+
+pub fn setup_logger(verbosity: u8, filter: &str) -> Result<()> {
+    use tracing::Level;
+    let log_level = match verbosity {
+        0 => Level::ERROR,
+        1 => Level::WARN,
+        2 => Level::INFO,
+        3 => Level::DEBUG,
+        _ => Level::TRACE,
+    };
+    let directive_1 = format!("{filter}={log_level}")
+        .parse()
+        .expect("valid log level");
+    let directive_2 = format!("webb_={log_level}")
+        .parse()
+        .expect("valid log level");
+    let env_filter = tracing_subscriber::EnvFilter::from_default_env()
+        .add_directive(directive_1)
+        .add_directive(directive_2);
+    let logger = tracing_subscriber::fmt()
+        .with_target(true)
+        .with_max_level(log_level)
+        .with_env_filter(env_filter);
+    let logger = logger.pretty();
+    logger.init();
+    Ok(())
+}

src/error.rs

@@ -79,7 +79,8 @@ impl<'r, 'o: 'r> Responder<'r, 'o> for Error {
                     },
                     Status::InternalServerError,
                 ),
-                webb_auth_sled::Error::InvalidId(_) => (
+                webb_auth_sled::Error::InvalidU65Id(_)
+                | webb_auth_sled::Error::InvalidI65Id(_) => (
                     ErrorResponse {
                         code: FaucetErrorCode::DataSerializationError,
                         message: self.to_string(),