david/nostr-rs-relay
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: format

Browse Source 
cargo fmt
This commit is contained in:
thesimplekid
2023-02-25 14:46:49 -06:00
committed by Greg Heartsfield
parent 440217e1ee
commit 6df92f9580
18 changed files with 500 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/repo/mod.rs
+2 -2
View File
@@ -7,10 +7,10 @@ use crate::utils::unix_time;
use async_trait::async_trait;
use rand::Rng;
pub mod sqlite;
pub mod sqlite_migration;
pub mod postgres;
pub mod postgres_migration;
pub mod sqlite;
pub mod sqlite_migration;
#[async_trait]
pub trait NostrRepo: Send + Sync {
src/repo/postgres.rs
+19 -18
View File
@@ -8,10 +8,11 @@ use async_std::stream::StreamExt;
use async_trait::async_trait;
use chrono::{DateTime, TimeZone, Utc};
use sqlx::postgres::PgRow;
use sqlx::Error::RowNotFound;
use sqlx::{Error, Execute, FromRow, Postgres, QueryBuilder, Row};
use std::time::{Duration, Instant};
use sqlx::Error::RowNotFound;
use crate::error;
use crate::hexrange::{hex_range, HexSearch};
use crate::repo::postgres_migration::run_migrations;
use crate::server::NostrMetrics;
@@ -19,8 +20,7 @@ use crate::utils::{is_hex, is_lower_hex, self};
use tokio::sync::mpsc::Sender;
use tokio::sync::oneshot::Receiver;
use tracing::log::trace;
use tracing::{debug, error, warn, info};
use crate::error;
use tracing::{debug, info, warn, error};
pub type PostgresPool = sqlx::pool::Pool<Postgres>;
@@ -78,7 +78,6 @@ async fn delete_expired(conn:PostgresPool) -> Result<u64> {
#[async_trait]
impl NostrRepo for PostgresRepo {
async fn start(&self) -> Result<()> {
// begin a cleanup task for expired events.
cleanup_expired(self.conn.clone(), Duration::from_secs(600)).await?;
@@ -116,7 +115,7 @@ impl NostrRepo for PostgresRepo {
}
}
if let Some(d_tag) = e.distinct_param() {
let repl_count:i64 = if is_lower_hex(&d_tag) && (d_tag.len() % 2 == 0) {
let repl_count: i64 = if is_lower_hex(&d_tag) && (d_tag.len() % 2 == 0) {
sqlx::query_scalar(
"SELECT count(*) AS count FROM event e LEFT JOIN tag t ON e.id=t.event_id WHERE e.pub_key=$1 AND e.kind=$2 AND t.name='d' AND t.value_hex=$3 AND e.created_at >= $4 LIMIT 1;")
.bind(hex::decode(&e.pubkey).ok())
@@ -139,7 +138,7 @@ impl NostrRepo for PostgresRepo {
// the same author/kind/tag value exist, and we can ignore
// this event.
if repl_count > 0 {
return Ok(0)
return Ok(0);
}
}
// ignore if the event hash is a duplicate.
@@ -159,7 +158,6 @@ ON CONFLICT (id) DO NOTHING"#,
.execute(&mut tx)
.await?
.rows_affected();
if ins_count == 0 {
// if the event was a duplicate, no need to insert event or
// pubkey references. This will abort the txn.
@@ -276,10 +274,10 @@ ON CONFLICT (id) DO NOTHING"#,
LEFT JOIN tag t ON e.id = t.event_id \
WHERE e.pub_key = $1 AND t.\"name\" = 'e' AND e.kind = 5 AND t.value = $2 LIMIT 1",
)
.bind(&pubkey_blob)
.bind(&id_blob)
.fetch_optional(&mut tx)
.await?;
.bind(&pubkey_blob)
.bind(&id_blob)
.fetch_optional(&mut tx)
.await?;
// check if a the query returned a result, meaning we should
// hid the current event
@@ -380,7 +378,10 @@ ON CONFLICT (id) DO NOTHING"#,
// check if this is still active; every 100 rows
if row_count % 100 == 0 && abandon_query_rx.try_recv().is_ok() {
debug!("query cancelled by client (cid: {}, sub: {:?})", client_id, sub.id);
debug!(
"query cancelled by client (cid: {}, sub: {:?})",
client_id, sub.id
);
return Ok(());
}
@@ -396,7 +397,10 @@ ON CONFLICT (id) DO NOTHING"#,
if last_successful_send + abort_cutoff < Instant::now() {
// the queue has been full for too long, abort
info!("aborting database query due to slow client");
metrics.query_aborts.with_label_values(&["slowclient"]).inc();
metrics
.query_aborts
.with_label_values(&["slowclient"])
.inc();
return Ok(());
}
// give the queue a chance to clear before trying again
@@ -467,9 +471,7 @@ ON CONFLICT (id) DO NOTHING"#,
let verify_time = now_jitter(600);
// update verification time and reset any failure count
sqlx::query(
"UPDATE user_verification SET verified_at = $1, fail_count = 0 WHERE id = $2",
)
sqlx::query("UPDATE user_verification SET verified_at = $1, fail_count = 0 WHERE id = $2")
.bind(Utc.timestamp_opt(verify_time as i64, 0).unwrap())
.bind(id as i64)
.execute(&self.conn)
@@ -767,8 +769,7 @@ fn query_from_filter(f: &ReqFilter) -> Option<QueryBuilder<Postgres>> {
impl FromRow<'_, PgRow> for VerificationRecord {
fn from_row(row: &'_ PgRow) -> std::result::Result<Self, Error> {
let name =
Nip05Name::try_from(row.get::<'_, &str, &str>("name")).or(Err(RowNotFound))?;
let name = Nip05Name::try_from(row.get::<'_, &str, &str>("name")).or(Err(RowNotFound))?;
Ok(VerificationRecord {
rowid: row.get::<'_, i64, &str>("id") as u64,
name,
src/repo/sqlite.rs
+101 -56
View File
@@ -1,32 +1,33 @@
//! Event persistence and querying
//use crate::config::SETTINGS;
use crate::config::Settings;
use crate::error::{Result,Error::SqlError};
use crate::db::QueryResult;
use crate::error::Result;
use crate::error::Error::SqlError;
use crate::event::{single_char_tagname, Event};
use crate::hexrange::hex_range;
use crate::hexrange::HexSearch;
use crate::repo::sqlite_migration::{STARTUP_SQL,upgrade_db};
use crate::utils::{is_hex,unix_time};
use crate::nip05::{Nip05Name, VerificationRecord};
use crate::subscription::{ReqFilter, Subscription};
use crate::repo::sqlite_migration::{upgrade_db, STARTUP_SQL};
use crate::server::NostrMetrics;
use crate::subscription::{ReqFilter, Subscription};
use crate::utils::{is_hex, unix_time};
use async_trait::async_trait;
use hex;
use r2d2;
use r2d2_sqlite::SqliteConnectionManager;
use rusqlite::params;
use rusqlite::types::ToSql;
use rusqlite::OpenFlags;
use tokio::sync::{Mutex, MutexGuard, Semaphore};
use std::fmt::Write as _;
use std::path::Path;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use std::time::Instant;
use tokio::sync::{Mutex, MutexGuard, Semaphore};
use tokio::task;
use tracing::{debug, info, trace, warn};
use async_trait::async_trait;
use crate::db::QueryResult;
use crate::repo::{now_jitter, NostrRepo};
@@ -54,7 +55,8 @@ pub struct SqliteRepo {
impl SqliteRepo {
// build all the pools needed
#[must_use] pub fn new(settings: &Settings, metrics: NostrMetrics) -> SqliteRepo {
#[must_use]
pub fn new(settings: &Settings, metrics: NostrMetrics) -> SqliteRepo {
let write_pool = build_pool(
"writer",
settings,
@@ -110,7 +112,8 @@ impl SqliteRepo {
// get relevant fields from event and convert to blobs.
let id_blob = hex::decode(&e.id).ok();
let pubkey_blob: Option<Vec<u8>> = hex::decode(&e.pubkey).ok();
let delegator_blob: Option<Vec<u8>> = e.delegated_by.as_ref().and_then(|d| hex::decode(d).ok());
let delegator_blob: Option<Vec<u8>> =
e.delegated_by.as_ref().and_then(|d| hex::decode(d).ok());
let event_str = serde_json::to_string(&e).ok();
// check for replaceable events that would hide this one; we won't even attempt to insert these.
if e.is_replaceable() {
@@ -130,7 +133,7 @@ impl SqliteRepo {
// the same author/kind/tag value exist, and we can ignore
// this event.
if repl_count.ok().is_some() {
return Ok(0)
return Ok(0);
}
}
// ignore if the event hash is a duplicate.
@@ -249,21 +252,24 @@ impl SqliteRepo {
#[async_trait]
impl NostrRepo for SqliteRepo {
async fn start(&self) -> Result<()> {
db_checkpoint_task(self.maint_pool.clone(), Duration::from_secs(60),
self.write_in_progress.clone(),
self.checkpoint_in_progress.clone()).await?;
cleanup_expired(self.maint_pool.clone(), Duration::from_secs(600),
self.write_in_progress.clone()).await
db_checkpoint_task(
self.maint_pool.clone(),
Duration::from_secs(60),
self.write_in_progress.clone(),
self.checkpoint_in_progress.clone()
).await?;
cleanup_expired(
self.maint_pool.clone(),
Duration::from_secs(600),
self.write_in_progress.clone()
).await
}
async fn migrate_up(&self) -> Result<usize> {
let _write_guard = self.write_in_progress.lock().await;
let mut conn = self.write_pool.get()?;
task::spawn_blocking(move || {
upgrade_db(&mut conn)
}).await?
task::spawn_blocking(move || upgrade_db(&mut conn)).await?
}
/// Persist event to database
async fn write_event(&self, e: &Event) -> Result<u64> {
@@ -322,9 +328,14 @@ impl NostrRepo for SqliteRepo {
// thread pool waiting for queries to finish under high load.
// Instead, don't bother spawning threads when they will just
// block on a database connection.
let sem = self.reader_threads_ready.clone().acquire_owned().await.unwrap();
let self=self.clone();
let metrics=self.metrics.clone();
let sem = self
.reader_threads_ready
.clone()
.acquire_owned()
.await
.unwrap();
let self = self.clone();
let metrics = self.metrics.clone();
task::spawn_blocking(move || {
{
// if we are waiting on a checkpoint, stop until it is complete
@@ -349,7 +360,10 @@ impl NostrRepo for SqliteRepo {
}
// check before getting a DB connection if the client still wants the results
if abandon_query_rx.try_recv().is_ok() {
debug!("query cancelled by client (before execution) (cid: {}, sub: {:?})", client_id, sub.id);
debug!(
"query cancelled by client (before execution) (cid: {}, sub: {:?})",
client_id, sub.id
);
return Ok(());
}
@@ -362,7 +376,9 @@ impl NostrRepo for SqliteRepo {
if let Ok(mut conn) = self.read_pool.get() {
{
let pool_state = self.read_pool.state();
metrics.db_connections.set((pool_state.connections - pool_state.idle_connections).into());
metrics
.db_connections
.set((pool_state.connections - pool_state.idle_connections).into());
}
for filter in sub.filters.iter() {
let filter_start = Instant::now();
@@ -379,7 +395,7 @@ impl NostrRepo for SqliteRepo {
let mut last_successful_send = Instant::now();
// execute the query.
// make the actual SQL query (with parameters inserted) available
conn.trace(Some(|x| {trace!("SQL trace: {:?}", x)}));
conn.trace(Some(|x| trace!("SQL trace: {:?}", x)));
let mut stmt = conn.prepare_cached(&q)?;
let mut event_rows = stmt.query(rusqlite::params_from_iter(p))?;
@@ -397,7 +413,10 @@ impl NostrRepo for SqliteRepo {
if slow_first_event && client_id.starts_with('0') {
debug!(
"filter first result in {:?} (slow): {} (cid: {}, sub: {:?})",
first_event_elapsed, serde_json::to_string(&filter)?, client_id, sub.id
first_event_elapsed,
serde_json::to_string(&filter)?,
client_id,
sub.id
);
}
first_result = false;
@@ -407,8 +426,14 @@ impl NostrRepo for SqliteRepo {
{
if self.checkpoint_in_progress.try_lock().is_err() {
// lock was held, abort this query
debug!("query aborted due to checkpoint (cid: {}, sub: {:?})", client_id, sub.id);
metrics.query_aborts.with_label_values(&["checkpoint"]).inc();
debug!(
"query aborted due to checkpoint (cid: {}, sub: {:?})",
client_id, sub.id
);
metrics
.query_aborts
.with_label_values(&["checkpoint"])
.inc();
return Ok(());
}
}
@@ -416,7 +441,10 @@ impl NostrRepo for SqliteRepo {
// check if this is still active; every 100 rows
if row_count % 100 == 0 && abandon_query_rx.try_recv().is_ok() {
debug!("query cancelled by client (cid: {}, sub: {:?})", client_id, sub.id);
debug!(
"query cancelled by client (cid: {}, sub: {:?})",
client_id, sub.id
);
return Ok(());
}
row_count += 1;
@@ -432,19 +460,31 @@ impl NostrRepo for SqliteRepo {
// the queue has been full for too long, abort
info!("aborting database query due to slow client (cid: {}, sub: {:?})",
client_id, sub.id);
metrics.query_aborts.with_label_values(&["slowclient"]).inc();
metrics
.query_aborts
.with_label_values(&["slowclient"])
.inc();
let ok: Result<()> = Ok(());
return ok;
}
// check if a checkpoint is trying to run, and abort
if self.checkpoint_in_progress.try_lock().is_err() {
// lock was held, abort this query
debug!("query aborted due to checkpoint (cid: {}, sub: {:?})", client_id, sub.id);
metrics.query_aborts.with_label_values(&["checkpoint"]).inc();
debug!(
"query aborted due to checkpoint (cid: {}, sub: {:?})",
client_id, sub.id
);
metrics
.query_aborts
.with_label_values(&["checkpoint"])
.inc();
return Ok(());
}
// give the queue a chance to clear before trying again
debug!("query thread sleeping due to full query_tx (cid: {}, sub: {:?})", client_id, sub.id);
debug!(
"query thread sleeping due to full query_tx (cid: {}, sub: {:?})",
client_id, sub.id
);
thread::sleep(Duration::from_millis(500));
}
// TODO: we could use try_send, but we'd have to juggle
@@ -465,10 +505,12 @@ impl NostrRepo for SqliteRepo {
if filter_start.elapsed() > slow_cutoff && client_id.starts_with('0') {
debug!(
"query filter req (slow): {} (cid: {}, sub: {:?}, filter: {})",
serde_json::to_string(&filter)?, client_id, sub.id, filter_count
serde_json::to_string(&filter)?,
client_id,
sub.id,
filter_count
);
}
}
} else {
warn!("Could not get a database connection for querying");
@@ -504,7 +546,8 @@ impl NostrRepo for SqliteRepo {
let start = Instant::now();
conn.execute_batch("PRAGMA optimize;").ok();
info!("optimize ran in {:?}", start.elapsed());
}).await?;
})
.await?;
Ok(())
}
@@ -557,8 +600,7 @@ impl NostrRepo for SqliteRepo {
let ok: Result<()> = Ok(());
ok
})
.await?
.await?
}
/// Update verification record as failed
@@ -596,7 +638,7 @@ impl NostrRepo for SqliteRepo {
let ok: Result<()> = Ok(());
ok
})
.await?
.await?
}
/// Get the latest verification record for a given pubkey.
@@ -684,14 +726,15 @@ fn override_index(f: &ReqFilter) -> Option<String> {
// queries for multiple kinds default to kind_index, which is
// significantly slower than kind_created_at_index.
if let Some(ks) = &f.kinds {
if f.ids.is_none() &&
ks.len() > 1 &&
f.since.is_none() &&
f.until.is_none() &&
f.tags.is_none() &&
f.authors.is_none() {
return Some("kind_created_at_index".into());
}
if f.ids.is_none()
&& ks.len() > 1
&& f.since.is_none()
&& f.until.is_none()
&& f.tags.is_none()
&& f.authors.is_none()
{
return Some("kind_created_at_index".into());
}
}
// if there is an author, it is much better to force the authors index.
if f.authors.is_some() {
@@ -726,7 +769,9 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
// check if the index needs to be overriden
let idx_name = override_index(f);
let idx_stmt = idx_name.as_ref().map_or_else(|| "".to_owned(), |i| format!("INDEXED BY {i}"));
let idx_stmt = idx_name
.as_ref()
.map_or_else(|| "".to_owned(), |i| format!("INDEXED BY {i}"));
let mut query = format!("SELECT e.content FROM event e {idx_stmt}");
// query parameters for SQLite
let mut params: Vec<Box<dyn ToSql>> = vec![];
@@ -744,9 +789,7 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
params.push(Box::new(ex));
}
Some(HexSearch::Range(lower, upper)) => {
auth_searches.push(
"(author>? AND author<?)".to_owned(),
);
auth_searches.push("(author>? AND author<?)".to_owned());
params.push(Box::new(lower));
params.push(Box::new(upper));
}
@@ -903,7 +946,7 @@ fn _query_from_sub(sub: &Subscription) -> (String, Vec<Box<dyn ToSql>>, Vec<Stri
.map(|s| format!("SELECT distinct content, created_at FROM ({s})"))
.collect();
let query: String = subqueries_selects.join(" UNION ");
(query, params,indexes)
(query, params, indexes)
}
/// Build a database connection pool.
@@ -1003,13 +1046,17 @@ pub fn delete_expired(conn: &mut PooledConnection) -> Result<usize> {
}
/// Perform database WAL checkpoint on a regular basis
pub async fn db_checkpoint_task(pool: SqlitePool, frequency: Duration, write_in_progress: Arc<Mutex<u64>>, checkpoint_in_progress: Arc<Mutex<u64>>) -> Result<()> {
pub async fn db_checkpoint_task(
pool: SqlitePool,
frequency: Duration,
write_in_progress: Arc<Mutex<u64>>,
checkpoint_in_progress: Arc<Mutex<u64>>) -> Result<()> {
// TODO; use acquire_many on the reader semaphore to stop them from interrupting this.
tokio::task::spawn(async move {
// WAL size in pages.
let mut current_wal_size = 0;
// WAL threshold for more aggressive checkpointing (10,000 pages, or about 40MB)
let wal_threshold = 1000*10;
let wal_threshold = 1000 * 10;
// default threshold for the busy timer
let busy_wait_default = Duration::from_secs(1);
// if the WAL file is getting too big, switch to this
@@ -1079,7 +1126,6 @@ pub fn checkpoint_db(conn: &mut PooledConnection) -> Result<usize> {
Ok(wal_size as usize)
}
/// Produce a arbitrary list of '?' parameters.
fn repeat_vars(count: usize) -> String {
if count == 0 {
@@ -1113,7 +1159,6 @@ fn log_pool_stats(name: &str, pool: &SqlitePool) {
);
}
/// Check if the pool is fully utilized
fn _pool_at_capacity(pool: &SqlitePool) -> bool {
let state: r2d2::State = pool.state();
src/repo/sqlite_migration.rs
+11 -7
View File
@@ -253,8 +253,8 @@ pub fn rebuild_tags(conn: &mut PooledConnection) -> Result<()> {
let mut stmt = tx.prepare("select id, content from event order by id;")?;
let mut tag_rows = stmt.query([])?;
while let Some(row) = tag_rows.next()? {
if (events_processed as f32)/(count as f32) > percent_done {
info!("Tag update {}% complete...", (100.0*percent_done).round());
if (events_processed as f32) / (count as f32) > percent_done {
info!("Tag update {}% complete...", (100.0 * percent_done).round());
percent_done += update_each_percent;
}
// we want to capture the event_id that had the tag, the tag name, and the tag hex value.
@@ -293,8 +293,6 @@ pub fn rebuild_tags(conn: &mut PooledConnection) -> Result<()> {
Ok(())
}
//// Migration Scripts
fn mig_1_to_2(conn: &mut PooledConnection) -> Result<usize> {
@@ -586,11 +584,17 @@ fn mig_11_to_12(conn: &mut PooledConnection) -> Result<usize> {
tx.execute("PRAGMA user_version = 12;", [])?;
}
tx.commit()?;
info!("database schema upgraded v11 -> v12 in {:?}", start.elapsed());
info!(
"database schema upgraded v11 -> v12 in {:?}",
start.elapsed()
);
// vacuum after large table modification
let start = Instant::now();
conn.execute("VACUUM;", [])?;
info!("vacuumed DB after hidden event cleanup in {:?}", start.elapsed());
info!(
"vacuumed DB after hidden event cleanup in {:?}",
start.elapsed()
);
Ok(12)
}
@@ -656,7 +660,7 @@ PRAGMA user_version = 15;
match conn.execute_batch(clear_hidden_sql) {
Ok(()) => {
info!("all hidden events removed");
},
}
Err(err) => {
error!("delete failed: {}", err);
panic!("could not remove hidden events");