From a4d384a8118e917e1e831351eac8fd0721ff0864 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Sun, 30 Jun 2024 14:10:16 +0800 Subject: [PATCH 1/9] add in table output. --- util/rich-indexer/resources/create_postgres_table.sql | 3 ++- util/rich-indexer/resources/create_sqlite_table.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/util/rich-indexer/resources/create_postgres_table.sql b/util/rich-indexer/resources/create_postgres_table.sql index f743425a06..26bdb5e721 100644 --- a/util/rich-indexer/resources/create_postgres_table.sql +++ b/util/rich-indexer/resources/create_postgres_table.sql @@ -58,7 +58,8 @@ CREATE TABLE output( capacity BIGINT NOT NULL, lock_script_id BIGINT, type_script_id BIGINT, - data BYTEA + data BYTEA, + is_spent BOOLEAN DEFAULT FALSE ); CREATE TABLE input( diff --git a/util/rich-indexer/resources/create_sqlite_table.sql b/util/rich-indexer/resources/create_sqlite_table.sql index ca85b20e5e..29c474414c 100644 --- a/util/rich-indexer/resources/create_sqlite_table.sql +++ b/util/rich-indexer/resources/create_sqlite_table.sql @@ -58,7 +58,8 @@ CREATE TABLE output( capacity INTEGER NOT NULL, lock_script_id INTEGER, type_script_id INTEGER, - data BLOB + data BLOB, + is_spent INTEGER DEFAULT 0 ); CREATE TABLE input( From 8a74491deb7f1151c6a519c16c69cf86ec212ecc Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Sun, 30 Jun 2024 15:15:59 +0800 Subject: [PATCH 2/9] update filed. --- util/rich-indexer/src/indexer/insert.rs | 26 +++++++++++++++++++++++++ util/rich-indexer/src/indexer/mod.rs | 3 +++ 2 files changed, 29 insertions(+) diff --git a/util/rich-indexer/src/indexer/insert.rs b/util/rich-indexer/src/indexer/insert.rs index 9c8831b347..765732100e 100644 --- a/util/rich-indexer/src/indexer/insert.rs +++ b/util/rich-indexer/src/indexer/insert.rs @@ -400,6 +400,32 @@ pub(crate) async fn bulk_insert_tx_association_cell_dep_table( .await } +pub(crate) async fn spend_cell( + out_point: &OutPoint, + tx: &mut Transaction<'_, Any>, +) -> Result { + let output_tx_hash = out_point.tx_hash().raw_data().to_vec(); + let output_index: u32 = out_point.index().unpack(); + + let updated_rows = sqlx::query( + r#" + UPDATE output + SET is_spent = 1 + WHERE + tx_id = (SELECT ckb_transaction.id FROM ckb_transaction WHERE tx_hash = $1) + AND output_index = $2 + "#, + ) + .bind(output_tx_hash) + .bind(output_index as i32) + .execute(&mut *tx) + .await + .map_err(|err| Error::DB(err.to_string()))? + .rows_affected(); + + Ok(updated_rows > 0) +} + pub(crate) async fn query_output_cell( out_point: &OutPoint, tx: &mut Transaction<'_, Any>, diff --git a/util/rich-indexer/src/indexer/mod.rs b/util/rich-indexer/src/indexer/mod.rs index cd5beb3a67..08f4cfcdb2 100644 --- a/util/rich-indexer/src/indexer/mod.rs +++ b/util/rich-indexer/src/indexer/mod.rs @@ -197,6 +197,9 @@ impl AsyncRichIndexer { if tx_index != 0 { for (input_index, input) in tx_view.inputs().into_iter().enumerate() { let out_point = input.previous_output(); + if spend_cell(&out_point, tx).await? { + break; + } if self.custom_filters.is_cell_filter_enabled() { if let Some((output_id, output, output_data)) = query_output_cell(&out_point, tx).await? From bdc77719b3053d40fdf595c8c8afb5b78475f25d Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Sun, 30 Jun 2024 17:53:48 +0800 Subject: [PATCH 3/9] when rollback. --- util/rich-indexer/src/indexer/mod.rs | 2 +- util/rich-indexer/src/indexer/remove.rs | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/util/rich-indexer/src/indexer/mod.rs b/util/rich-indexer/src/indexer/mod.rs index 08f4cfcdb2..a244fc36e0 100644 --- a/util/rich-indexer/src/indexer/mod.rs +++ b/util/rich-indexer/src/indexer/mod.rs @@ -197,7 +197,7 @@ impl AsyncRichIndexer { if tx_index != 0 { for (input_index, input) in tx_view.inputs().into_iter().enumerate() { let out_point = input.previous_output(); - if spend_cell(&out_point, tx).await? { + if !spend_cell(&out_point, tx).await? { break; } if self.custom_filters.is_cell_filter_enabled() { diff --git a/util/rich-indexer/src/indexer/remove.rs b/util/rich-indexer/src/indexer/remove.rs index 56114f4c32..7f4bf24f4c 100644 --- a/util/rich-indexer/src/indexer/remove.rs +++ b/util/rich-indexer/src/indexer/remove.rs @@ -14,6 +14,9 @@ pub(crate) async fn rollback_block(tx: &mut Transaction<'_, Any>) -> Result<(), let tx_id_list = query_tx_id_list_by_block_id(block_id, tx).await?; let output_lock_type_list = query_outputs_by_tx_id_list(&tx_id_list, tx).await?; + // update spent cells + reset_spent_cells(&tx_id_list, tx).await?; + // remove transactions, associations, inputs, output remove_batch_by_blobs("ckb_transaction", "id", &tx_id_list, tx).await?; remove_batch_by_blobs("tx_association_cell_dep", "tx_id", &tx_id_list, tx).await?; @@ -80,6 +83,28 @@ async fn remove_batch_by_blobs( Ok(()) } +async fn reset_spent_cells(tx_id_list: &[i64], tx: &mut Transaction<'_, Any>) -> Result<(), Error> { + let query = SqlBuilder::update_table("output") + .set("is_spent", 0) + .and_where_in_query( + "id", + SqlBuilder::select_from("input") + .field("output_id") + .and_where_in("consumed_tx_id", tx_id_list) + .query() + .map_err(|err| Error::DB(err.to_string()))?, + ) + .sql() + .map_err(|err| Error::DB(err.to_string()))?; + + sqlx::query(&query) + .execute(&mut *tx) + .await + .map_err(|err| Error::DB(err.to_string()))?; + + Ok(()) +} + async fn query_uncle_id_list_by_block_id( block_id: i64, tx: &mut Transaction<'_, Any>, From 120f806163529f3500a0c3f6eb3cda30f565060a Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Sun, 30 Jun 2024 18:30:46 +0800 Subject: [PATCH 4/9] update get_cells --- .../src/indexer_handle/async_indexer_handle/get_cells.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells.rs b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells.rs index 63a89d4800..1e6c158ab6 100644 --- a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells.rs +++ b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells.rs @@ -98,9 +98,7 @@ impl AsyncRichIndexerHandle { .join(name!("script";"lock_script")) .on("output.lock_script_id = lock_script.id"), } - .join("input") - .on("output.id = input.output_id") - .and_where("input.output_id IS NULL"); // live cells + .and_where("output.is_spent = 0"); // live cells // filter cells in pool let mut dead_cells = Vec::new(); From bf483bf39f64d61ff68858ad2da58c08e1a604a9 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Sun, 30 Jun 2024 18:31:09 +0800 Subject: [PATCH 5/9] update get_cells_capacity --- .../async_indexer_handle/get_cells_capacity.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells_capacity.rs b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells_capacity.rs index 5a14d90592..5c5aee71fe 100644 --- a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells_capacity.rs +++ b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells_capacity.rs @@ -51,27 +51,25 @@ impl AsyncRichIndexerHandle { .join("ckb_transaction") .on("output.tx_id = ckb_transaction.id"); } - query_builder - .left() - .join("input") - .on("output.id = input.output_id"); if let Some(ref filter) = search_key.filter { if filter.script.is_some() || filter.script_len_range.is_some() { match search_key.script_type { IndexerScriptType::Lock => { query_builder + .left() .join(name!("script";"type_script")) .on("output.type_script_id = type_script.id"); } IndexerScriptType::Type => { query_builder + .left() .join(name!("script";"lock_script")) .on("output.lock_script_id = lock_script.id"); } } } } - query_builder.and_where("input.output_id IS NULL"); // live cells + query_builder.and_where("output.is_spent = 0"); // live cells // filter cells in pool let mut dead_cells = Vec::new(); From 1cde270b4ca1db22b08a52628a4660349e35e1c2 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Sun, 30 Jun 2024 23:59:12 +0800 Subject: [PATCH 6/9] add migration sql. --- .../resources/create_postgres_table.sql | 3 +- .../resources/create_sqlite_table.sql | 3 +- .../20240603_add_is_spent_to_output.sql | 9 ++++++ util/rich-indexer/src/store.rs | 31 ++++++++++--------- 4 files changed, 28 insertions(+), 18 deletions(-) create mode 100644 util/rich-indexer/resources/migrations/20240603_add_is_spent_to_output.sql diff --git a/util/rich-indexer/resources/create_postgres_table.sql b/util/rich-indexer/resources/create_postgres_table.sql index 26bdb5e721..f743425a06 100644 --- a/util/rich-indexer/resources/create_postgres_table.sql +++ b/util/rich-indexer/resources/create_postgres_table.sql @@ -58,8 +58,7 @@ CREATE TABLE output( capacity BIGINT NOT NULL, lock_script_id BIGINT, type_script_id BIGINT, - data BYTEA, - is_spent BOOLEAN DEFAULT FALSE + data BYTEA ); CREATE TABLE input( diff --git a/util/rich-indexer/resources/create_sqlite_table.sql b/util/rich-indexer/resources/create_sqlite_table.sql index 29c474414c..ca85b20e5e 100644 --- a/util/rich-indexer/resources/create_sqlite_table.sql +++ b/util/rich-indexer/resources/create_sqlite_table.sql @@ -58,8 +58,7 @@ CREATE TABLE output( capacity INTEGER NOT NULL, lock_script_id INTEGER, type_script_id INTEGER, - data BLOB, - is_spent INTEGER DEFAULT 0 + data BLOB ); CREATE TABLE input( diff --git a/util/rich-indexer/resources/migrations/20240603_add_is_spent_to_output.sql b/util/rich-indexer/resources/migrations/20240603_add_is_spent_to_output.sql new file mode 100644 index 0000000000..a7cd72d02c --- /dev/null +++ b/util/rich-indexer/resources/migrations/20240603_add_is_spent_to_output.sql @@ -0,0 +1,9 @@ +-- 20240630_add_is_spent_to_output.sql + +ALTER TABLE output +ADD COLUMN is_spent INTEGER DEFAULT 0; + +UPDATE output +SET is_spent = 1 +FROM input +WHERE input.output_id = output.id; diff --git a/util/rich-indexer/src/store.rs b/util/rich-indexer/src/store.rs index d9caa7f6fe..4ac79af061 100644 --- a/util/rich-indexer/src/store.rs +++ b/util/rich-indexer/src/store.rs @@ -5,13 +5,14 @@ use log::LevelFilter; use once_cell::sync::OnceCell; use sqlx::{ any::{Any, AnyArguments, AnyConnectOptions, AnyPool, AnyPoolOptions, AnyRow}, + migrate::Migrator, query::{Query, QueryAs}, ConnectOptions, IntoArguments, Row, Transaction, }; use std::fs::OpenOptions; use std::marker::{Send, Unpin}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::str::FromStr; use std::{fmt::Debug, sync::Arc, time::Duration}; @@ -20,6 +21,7 @@ const SQL_SQLITE_CREATE_TABLE: &str = include_str!("../resources/create_sqlite_t const SQL_SQLITE_CREATE_INDEX: &str = include_str!("../resources/create_sqlite_index.sql"); const SQL_POSTGRES_CREATE_TABLE: &str = include_str!("../resources/create_postgres_table.sql"); const SQL_POSTGRES_CREATE_INDEX: &str = include_str!("../resources/create_postgres_index.sql"); +const SQL_MIGRATIONS: &str = "./util/rich-indexer/resources/migrations"; #[derive(Clone, Default)] pub struct SQLXPool { @@ -36,13 +38,6 @@ impl Debug for SQLXPool { } impl SQLXPool { - pub fn new() -> Self { - SQLXPool { - pool: Arc::new(OnceCell::new()), - db_driver: DBDriver::default(), - } - } - pub async fn connect(&mut self, db_config: &RichIndexerConfig) -> Result<()> { let pool_options = AnyPoolOptions::new() .max_connections(10) @@ -50,7 +45,7 @@ impl SQLXPool { .acquire_timeout(Duration::from_secs(60)) .max_lifetime(Duration::from_secs(1800)) .idle_timeout(Duration::from_secs(30)); - match db_config.db_type { + let pool = match db_config.db_type { DBDriver::Sqlite => { let require_init = is_sqlite_require_init(db_config); let uri = build_url_for_sqlite(db_config); @@ -59,13 +54,13 @@ impl SQLXPool { let pool = pool_options.connect_with(connection_options).await?; log::info!("SQLite is connected."); self.pool - .set(pool) + .set(pool.clone()) .map_err(|_| anyhow!("set pool failed!"))?; if require_init { self.create_tables_for_sqlite().await?; } self.db_driver = DBDriver::Sqlite; - Ok(()) + pool } DBDriver::Postgres => { let require_init = self.is_postgres_require_init(db_config).await?; @@ -75,15 +70,23 @@ impl SQLXPool { let pool = pool_options.connect_with(connection_options).await?; log::info!("PostgreSQL is connected."); self.pool - .set(pool) + .set(pool.clone()) .map_err(|_| anyhow!("set pool failed"))?; if require_init { self.create_tables_for_postgres().await?; } self.db_driver = DBDriver::Postgres; - Ok(()) + pool } - } + }; + + // Run migrations + log::info!("Running migrations..."); + let migrator = Migrator::new(Path::new(SQL_MIGRATIONS)).await?; + migrator.run(&pool).await?; + log::info!("Migrations are done."); + + Ok(()) } pub async fn fetch_count(&self, table_name: &str) -> Result { From 99788643b9070848332c893c04a87e6e5a921d0b Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Tue, 2 Jul 2024 20:18:42 +0800 Subject: [PATCH 7/9] fix ci unit test. --- util/app-config/src/configs/rich_indexer.rs | 9 +++++++++ util/rich-indexer/src/store.rs | 5 ++--- util/rich-indexer/src/tests/mod.rs | 4 ++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/util/app-config/src/configs/rich_indexer.rs b/util/app-config/src/configs/rich_indexer.rs index bdf9a2fd5a..65cd7c5a7b 100644 --- a/util/app-config/src/configs/rich_indexer.rs +++ b/util/app-config/src/configs/rich_indexer.rs @@ -3,6 +3,7 @@ use std::{default::Default, path::PathBuf}; const PGSQL: &str = "postgres://"; const SQLITE: &str = "sqlite://"; +const SQL_MIGRATIONS_PATH: &str = "./util/rich-indexer/resources/migrations"; /// Rich indexer database type. #[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq, Eq, Copy)] @@ -49,6 +50,9 @@ pub struct RichIndexerConfig { /// The database password. #[serde(default = "default_db_password")] pub db_password: String, + /// The migrations path with a default value + #[serde(default = "default_migrations_path", skip_deserializing)] + pub migrations_path: PathBuf, } impl Default for RichIndexerConfig { @@ -61,6 +65,7 @@ impl Default for RichIndexerConfig { db_port: default_db_port(), db_user: default_db_user(), db_password: default_db_password(), + migrations_path: default_migrations_path(), } } } @@ -84,3 +89,7 @@ fn default_db_user() -> String { fn default_db_password() -> String { "123456".to_string() } + +fn default_migrations_path() -> PathBuf { + PathBuf::from(SQL_MIGRATIONS_PATH) +} diff --git a/util/rich-indexer/src/store.rs b/util/rich-indexer/src/store.rs index 4ac79af061..d6626de3cc 100644 --- a/util/rich-indexer/src/store.rs +++ b/util/rich-indexer/src/store.rs @@ -12,7 +12,7 @@ use sqlx::{ use std::fs::OpenOptions; use std::marker::{Send, Unpin}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::str::FromStr; use std::{fmt::Debug, sync::Arc, time::Duration}; @@ -21,7 +21,6 @@ const SQL_SQLITE_CREATE_TABLE: &str = include_str!("../resources/create_sqlite_t const SQL_SQLITE_CREATE_INDEX: &str = include_str!("../resources/create_sqlite_index.sql"); const SQL_POSTGRES_CREATE_TABLE: &str = include_str!("../resources/create_postgres_table.sql"); const SQL_POSTGRES_CREATE_INDEX: &str = include_str!("../resources/create_postgres_index.sql"); -const SQL_MIGRATIONS: &str = "./util/rich-indexer/resources/migrations"; #[derive(Clone, Default)] pub struct SQLXPool { @@ -82,7 +81,7 @@ impl SQLXPool { // Run migrations log::info!("Running migrations..."); - let migrator = Migrator::new(Path::new(SQL_MIGRATIONS)).await?; + let migrator = Migrator::new(db_config.migrations_path.clone()).await?; migrator.run(&pool).await?; log::info!("Migrations are done."); diff --git a/util/rich-indexer/src/tests/mod.rs b/util/rich-indexer/src/tests/mod.rs index 5493fe5291..4cdb9a963e 100644 --- a/util/rich-indexer/src/tests/mod.rs +++ b/util/rich-indexer/src/tests/mod.rs @@ -14,13 +14,17 @@ use ckb_jsonrpc_types::{ use ckb_types::h256; use ckb_types::prelude::*; +use std::path::PathBuf; + const MEMORY_DB: &str = ":memory:"; const BLOCK_DIR: &str = "./src/tests/data/blocks/"; +const MIGRATIONS_PATH: &str = "./resources/migrations"; async fn connect_sqlite(store_path: &str) -> SQLXPool { let mut pool = SQLXPool::default(); let config = RichIndexerConfig { store: store_path.into(), + migrations_path: PathBuf::from(MIGRATIONS_PATH), ..Default::default() }; pool.connect(&config).await.unwrap(); From eb72c29b2ddd7e0b4ff399a42d15a04032b62e80 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Fri, 26 Jul 2024 13:13:06 +0800 Subject: [PATCH 8/9] fix: can't find migrations dir. --- Cargo.lock | 21 +++++++++++++++++++++ util/app-config/src/configs/rich_indexer.rs | 9 --------- util/rich-indexer/Cargo.toml | 2 ++ util/rich-indexer/src/store.rs | 16 ++++++++++++++-- util/rich-indexer/src/tests/mod.rs | 4 ---- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c5db05f2e7..40f03b70a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1388,6 +1388,7 @@ dependencies = [ "ckb-types", "futures", "hex", + "include_dir", "log", "num-bigint", "once_cell", @@ -1395,6 +1396,7 @@ dependencies = [ "serde_json", "sql-builder", "sqlx", + "tempfile", "tokio", ] @@ -3112,6 +3114,25 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "includedir" version = "0.6.0" diff --git a/util/app-config/src/configs/rich_indexer.rs b/util/app-config/src/configs/rich_indexer.rs index 65cd7c5a7b..bdf9a2fd5a 100644 --- a/util/app-config/src/configs/rich_indexer.rs +++ b/util/app-config/src/configs/rich_indexer.rs @@ -3,7 +3,6 @@ use std::{default::Default, path::PathBuf}; const PGSQL: &str = "postgres://"; const SQLITE: &str = "sqlite://"; -const SQL_MIGRATIONS_PATH: &str = "./util/rich-indexer/resources/migrations"; /// Rich indexer database type. #[derive(Clone, Debug, Serialize, Deserialize, Default, PartialEq, Eq, Copy)] @@ -50,9 +49,6 @@ pub struct RichIndexerConfig { /// The database password. #[serde(default = "default_db_password")] pub db_password: String, - /// The migrations path with a default value - #[serde(default = "default_migrations_path", skip_deserializing)] - pub migrations_path: PathBuf, } impl Default for RichIndexerConfig { @@ -65,7 +61,6 @@ impl Default for RichIndexerConfig { db_port: default_db_port(), db_user: default_db_user(), db_password: default_db_password(), - migrations_path: default_migrations_path(), } } } @@ -89,7 +84,3 @@ fn default_db_user() -> String { fn default_db_password() -> String { "123456".to_string() } - -fn default_migrations_path() -> PathBuf { - PathBuf::from(SQL_MIGRATIONS_PATH) -} diff --git a/util/rich-indexer/Cargo.toml b/util/rich-indexer/Cargo.toml index e43297572e..5633c510ff 100644 --- a/util/rich-indexer/Cargo.toml +++ b/util/rich-indexer/Cargo.toml @@ -24,6 +24,8 @@ num-bigint = "0.4" once_cell = "1.8.0" sql-builder = "3.1" sqlx = { version = "0.6", features = ["runtime-tokio-rustls", "any", "sqlite", "postgres"] } +include_dir = "0.7" +tempfile = "3" [dev-dependencies] hex = "0.4" diff --git a/util/rich-indexer/src/store.rs b/util/rich-indexer/src/store.rs index d6626de3cc..22c910ff2a 100644 --- a/util/rich-indexer/src/store.rs +++ b/util/rich-indexer/src/store.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, Result}; use ckb_app_config::{DBDriver, RichIndexerConfig}; use futures::TryStreamExt; +use include_dir::{include_dir, Dir}; use log::LevelFilter; use once_cell::sync::OnceCell; use sqlx::{ @@ -9,8 +10,9 @@ use sqlx::{ query::{Query, QueryAs}, ConnectOptions, IntoArguments, Row, Transaction, }; +use tempfile::tempdir; -use std::fs::OpenOptions; +use std::fs::{self, OpenOptions}; use std::marker::{Send, Unpin}; use std::path::PathBuf; use std::str::FromStr; @@ -21,6 +23,7 @@ const SQL_SQLITE_CREATE_TABLE: &str = include_str!("../resources/create_sqlite_t const SQL_SQLITE_CREATE_INDEX: &str = include_str!("../resources/create_sqlite_index.sql"); const SQL_POSTGRES_CREATE_TABLE: &str = include_str!("../resources/create_postgres_table.sql"); const SQL_POSTGRES_CREATE_INDEX: &str = include_str!("../resources/create_postgres_index.sql"); +static MIGRATIONS_DIR: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/resources/migrations"); #[derive(Clone, Default)] pub struct SQLXPool { @@ -81,7 +84,16 @@ impl SQLXPool { // Run migrations log::info!("Running migrations..."); - let migrator = Migrator::new(db_config.migrations_path.clone()).await?; + let temp_dir = tempdir()?; + for file in MIGRATIONS_DIR.files() { + log::info!("Found migration file: {:?}", file.path()); + let file_path = temp_dir.path().join(file.path()); + if let Some(parent) = file_path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(&file_path, file.contents())?; + } + let migrator = Migrator::new(temp_dir.path()).await?; migrator.run(&pool).await?; log::info!("Migrations are done."); diff --git a/util/rich-indexer/src/tests/mod.rs b/util/rich-indexer/src/tests/mod.rs index 4cdb9a963e..5493fe5291 100644 --- a/util/rich-indexer/src/tests/mod.rs +++ b/util/rich-indexer/src/tests/mod.rs @@ -14,17 +14,13 @@ use ckb_jsonrpc_types::{ use ckb_types::h256; use ckb_types::prelude::*; -use std::path::PathBuf; - const MEMORY_DB: &str = ":memory:"; const BLOCK_DIR: &str = "./src/tests/data/blocks/"; -const MIGRATIONS_PATH: &str = "./resources/migrations"; async fn connect_sqlite(store_path: &str) -> SQLXPool { let mut pool = SQLXPool::default(); let config = RichIndexerConfig { store: store_path.into(), - migrations_path: PathBuf::from(MIGRATIONS_PATH), ..Default::default() }; pool.connect(&config).await.unwrap(); From 1f6d52126ddbe2c8d969d8fd727c1ab95e03c9f9 Mon Sep 17 00:00:00 2001 From: EthanYuan Date: Thu, 22 Aug 2024 09:43:45 +0800 Subject: [PATCH 9/9] add ignore item in deny.toml. --- deny.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deny.toml b/deny.toml index aadefc53d2..fabff48ef2 100644 --- a/deny.toml +++ b/deny.toml @@ -70,6 +70,10 @@ feature-depth = 1 # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ +# https://rustsec.org/advisories/RUSTSEC-2024-0363 +# https://github.com/launchbadge/sqlx/issues/3440 +# The queries for the rich indexer receive input parameters via RPC, and the data size is far less than 4GB, so this issue can be temporarily ignored while waiting for sqlx to be fixed. + "RUSTSEC-2024-0363", # https://rustsec.org/advisories/RUSTSEC-2022-0090 # It was sometimes possible for SQLite versions >= 1.0.12, < 3.39.2 to allow an array-bounds overflow when large string were input into SQLite's `printf` function. "RUSTSEC-2022-0090",