Compare commits

...
18 Commits
Author SHA1 Message Date
Greg Heartsfield 6a8c4ed1b5 build: bump version to 0.7.14 2022-12-26 11:26:48 -06:00
Greg Heartsfield 966c853700 docs: non-docker quick start 2022-12-26 10:34:09 -06:00
Greg Heartsfield 65fd0ed08b feat: increase wal_checkpoint time when WAL is large 2022-12-26 10:03:51 -06:00
Greg Heartsfield 0b51675b38 improvement: change suggestion and default for max sqlite DB readers 2022-12-25 11:17:08 -06:00
Greg Heartsfield 2e22334631 refactor: formatting 2022-12-25 11:06:30 -06:00
Greg Heartsfield cb2ac4bf0f improvement: give threads unique names 2022-12-25 10:47:32 -06:00
Greg Heartsfield 38dc7789dc improvement: cleaner slow query logs 2022-12-25 10:47:32 -06:00
Greg Heartsfield ce0e00ffb3 feat: log reader DB pool stats every minute 2022-12-25 10:47:32 -06:00
Greg Heartsfield 3e4ae4aeec feat: cache prepared statements and trace expanded SQL queries 2022-12-25 10:47:32 -06:00
Greg Heartsfield c6a8807485 improvement: send error on empty-string prefix author/id searches 2022-12-25 10:47:32 -06:00
Greg Heartsfield 8137b6211c refactor: clippy suggestions 2022-12-24 10:29:47 -06:00
Greg Heartsfield 29effaae23 build: remove pre-commit rustfmt check 2022-12-24 10:29:30 -06:00
Greg Heartsfield e5074f2e46 feat(NIP-28): replaceable kind 41 channel metadata events 2022-12-24 10:14:43 -06:00
Blake Jakopovic 4fd7643907 feat: change pub(crate) to pub for use as a library 2022-12-23 07:14:58 -06:00
Greg Heartsfield 1e1ec69175 build: remove unnecessary dockerfile mod script 2022-12-23 06:52:09 -06:00
benthecarman e08647867c refactor: remove code duplication for simple_event 2022-12-23 06:39:50 -06:00
Greg Heartsfield ae0f7171ed build: remove digest-locked docker base images 2022-12-23 06:30:59 -06:00
Greg Heartsfield 4f1a912f36 feat: log origin header from websocket requests
fixes: https://todo.sr.ht/~gheartsfield/nostr-rs-relay/33
2022-12-22 16:55:53 -06:00
13 changed files with 229 additions and 130 deletions
+1 -1
View File
@@ -11,6 +11,6 @@ repos:
- repo: https://github.com/doublify/pre-commit-rust - repo: https://github.com/doublify/pre-commit-rust
rev: v1.0 rev: v1.0
hooks: hooks:
- id: fmt # - id: fmt
- id: cargo-check - id: cargo-check
- id: clippy - id: clippy
Generated
+1 -1
View File
@@ -1096,7 +1096,7 @@ checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
[[package]] [[package]]
name = "nostr-rs-relay" name = "nostr-rs-relay"
version = "0.7.13" version = "0.7.14"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bitcoin_hashes", "bitcoin_hashes",
+2 -2
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "nostr-rs-relay" name = "nostr-rs-relay"
version = "0.7.13" version = "0.7.14"
edition = "2021" edition = "2021"
authors = ["Greg Heartsfield <scsibug@imap.cc>"] authors = ["Greg Heartsfield <scsibug@imap.cc>"]
description = "A relay implementation for the Nostr protocol" description = "A relay implementation for the Nostr protocol"
@@ -28,7 +28,7 @@ secp256k1 = {version = "0.21", features = ["rand", "rand-std", "serde", "bitcoin
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = {version = "1.0", features = ["preserve_order"]} serde_json = {version = "1.0", features = ["preserve_order"]}
hex = "0.4" hex = "0.4"
rusqlite = { version = "0.26", features = ["limits","bundled"]} rusqlite = { version = "0.26", features = ["limits","bundled","modern_sqlite", "trace"]}
r2d2 = "0.8" r2d2 = "0.8"
r2d2_sqlite = "0.19" r2d2_sqlite = "0.19"
lazy_static = "1.4" lazy_static = "1.4"
+2 -2
View File
@@ -1,4 +1,4 @@
FROM docker.io/library/rust:1.66.0@sha256:359949280cebefe93ccb33089fe25111a3aadfe99eac4b6cbe8ec3e1b571dacb as builder FROM docker.io/library/rust:1.66.0 as builder
RUN USER=root cargo install cargo-auditable RUN USER=root cargo install cargo-auditable
RUN USER=root cargo new --bin nostr-rs-relay RUN USER=root cargo new --bin nostr-rs-relay
@@ -17,7 +17,7 @@ COPY ./src ./src
RUN rm ./target/release/deps/nostr*relay* RUN rm ./target/release/deps/nostr*relay*
RUN cargo auditable build --release --locked RUN cargo auditable build --release --locked
FROM docker.io/library/debian:bullseye-20221205-slim@sha256:25f10b4f1ded5341a3ca0a30290ff3cd5639415f0c5a2222d5e7d5dd72952aa1 FROM docker.io/library/debian:bullseye-slim
ARG APP=/usr/src/app ARG APP=/usr/src/app
ARG APP_DATA=/usr/src/app/db ARG APP_DATA=/usr/src/app/db
+33
View File
@@ -29,6 +29,7 @@ mirrored on [GitHub](https://github.com/scsibug/nostr-rs-relay).
- [x] NIP-20: [Command Results](https://github.com/nostr-protocol/nips/blob/master/20.md) - [x] NIP-20: [Command Results](https://github.com/nostr-protocol/nips/blob/master/20.md)
- [x] NIP-22: [Event `created_at` limits](https://github.com/nostr-protocol/nips/blob/master/22.md) (_future-dated events only_) - [x] NIP-22: [Event `created_at` limits](https://github.com/nostr-protocol/nips/blob/master/22.md) (_future-dated events only_)
- [x] NIP-26: [Event Delegation](https://github.com/nostr-protocol/nips/blob/master/26.md) - [x] NIP-26: [Event Delegation](https://github.com/nostr-protocol/nips/blob/master/26.md)
- [x] NIP-28: [Public Chat](https://github.com/nostr-protocol/nips/blob/master/28.md)
## Quick Start ## Quick Start
@@ -81,6 +82,38 @@ Text Note [81cf...2652] from 296a...9b92 5 seconds ago
A pre-built container is also available on DockerHub: A pre-built container is also available on DockerHub:
https://hub.docker.com/r/scsibug/nostr-rs-relay https://hub.docker.com/r/scsibug/nostr-rs-relay
## Build and Run (without Docker)
Building `nostr-rs-relay` requires an installation of Cargo & Rust: https://www.rust-lang.org/tools/install
Clone this repository, and then build a release version of the relay:
```console
$ git clone -q https://git.sr.ht/\~gheartsfield/nostr-rs-relay
$ cd nostr-rs-relay
$ cargo build -q -r
```
The relay executable is now located in
`target/release/nostr-rs-relay`. In order to run it with logging
enabled, execute it with the `RUST_LOG` variable set:
```console
$ RUST_LOG=warn,nostr_rs_relay=info ./target/release/nostr-rs-relay
Dec 26 10:31:56.455 INFO nostr_rs_relay: Starting up from main
Dec 26 10:31:56.464 INFO nostr_rs_relay::server: listening on: 0.0.0.0:8080
Dec 26 10:31:56.466 INFO nostr_rs_relay::server: db writer created
Dec 26 10:31:56.466 INFO nostr_rs_relay::db: Built a connection pool "event writer" (min=1, max=2)
Dec 26 10:31:56.466 INFO nostr_rs_relay::db: opened database "./nostr.db" for writing
Dec 26 10:31:56.466 INFO nostr_rs_relay::schema: DB version = 11
Dec 26 10:31:56.467 INFO nostr_rs_relay::db: Built a connection pool "maintenance writer" (min=1, max=2)
Dec 26 10:31:56.467 INFO nostr_rs_relay::server: control message listener started
Dec 26 10:31:56.468 INFO nostr_rs_relay::db: Built a connection pool "client query" (min=4, max=8)
```
You now have a running relay, on port `8080`. Use a `nostr` client or
`websocat` to connect and send/query for events.
## Configuration ## Configuration
The sample [`config.toml`](config.toml) file demonstrates the The sample [`config.toml`](config.toml) file demonstrates the
+3 -2
View File
@@ -36,8 +36,9 @@ data_directory = "."
# Minimum number of SQLite reader connections # Minimum number of SQLite reader connections
#min_conn = 4 #min_conn = 4
# Maximum number of SQLite reader connections # Maximum number of SQLite reader connections. Recommend setting this
#max_conn = 128 # to approx the number of cores.
#max_conn = 8
[network] [network]
# Bind to this network address # Bind to this network address
-3
View File
@@ -1,3 +0,0 @@
#!/usr/bin/env bash
sed -E 's/@sha256:[[:alnum:]]+//g' Dockerfile > Dockerfile.any-platform
echo "Created platform-agnostic Dockerfile in 'Dockerfile.any-platform'"
+1 -1
View File
@@ -207,7 +207,7 @@ impl Default for Settings {
data_directory: ".".to_owned(), data_directory: ".".to_owned(),
in_memory: false, in_memory: false,
min_conn: 4, min_conn: 4,
max_conn: 128, max_conn: 8,
}, },
network: Network { network: Network {
port: 8080, port: 8080,
+76 -51
View File
@@ -39,9 +39,8 @@ pub struct SubmittedEvent {
/// Database file /// Database file
pub const DB_FILE: &str = "nostr.db"; pub const DB_FILE: &str = "nostr.db";
/// How frequently to run maintenance /// How frequently to attempt checkpointing
/// How many persisted events before DB maintenannce is triggered. pub const CHECKPOINT_FREQ_SEC: u64 = 60;
pub const EVENT_MAINTENANCE_FREQ_SEC: u64 = 60;
/// How many persisted events before we pause for backups. /// How many persisted events before we pause for backups.
/// It isn't clear this is enough to make the online backup API work yet. /// It isn't clear this is enough to make the online backup API work yet.
@@ -94,6 +93,16 @@ pub fn build_pool(
pool pool
} }
/// Display database pool stats every 1 minute
pub async fn monitor_pool(name: &str, pool: SqlitePool) {
let sleep_dur = Duration::from_secs(60);
loop {
log_pool_stats(name, &pool);
tokio::time::sleep(sleep_dur).await;
}
}
/// Perform normal maintenance /// Perform normal maintenance
pub fn optimize_db(conn: &mut PooledConnection) -> Result<()> { pub fn optimize_db(conn: &mut PooledConnection) -> Result<()> {
let start = Instant::now(); let start = Instant::now();
@@ -102,15 +111,15 @@ pub fn optimize_db(conn: &mut PooledConnection) -> Result<()> {
Ok(()) Ok(())
} }
#[derive(Debug)] #[derive(Debug)]
enum SqliteReturnStatus { enum SqliteStatus {
SqliteOk, Ok,
SqliteBusy, Busy,
SqliteError, Error,
SqliteOther(u64), Other(u64),
} }
/// Checkpoint/Truncate WAL /// Checkpoint/Truncate WAL. Returns the number of WAL pages remaining.
pub fn checkpoint_db(conn: &mut PooledConnection) -> Result<()> { pub fn checkpoint_db(conn: &mut PooledConnection) -> Result<usize> {
let query = "PRAGMA wal_checkpoint(TRUNCATE);"; let query = "PRAGMA wal_checkpoint(TRUNCATE);";
let start = Instant::now(); let start = Instant::now();
let (cp_result, wal_size, _frames_checkpointed) = conn.query_row(query, [], |row| { let (cp_result, wal_size, _frames_checkpointed) = conn.query_row(query, [], |row| {
@@ -120,10 +129,10 @@ pub fn checkpoint_db(conn: &mut PooledConnection) -> Result<()> {
Ok((checkpoint_result, wal_size, frames_checkpointed)) Ok((checkpoint_result, wal_size, frames_checkpointed))
})?; })?;
let result = match cp_result { let result = match cp_result {
0 => SqliteReturnStatus::SqliteOk, 0 => SqliteStatus::Ok,
1 => SqliteReturnStatus::SqliteBusy, 1 => SqliteStatus::Busy,
2 => SqliteReturnStatus::SqliteError, 2 => SqliteStatus::Error,
x => SqliteReturnStatus::SqliteOther(x), x => SqliteStatus::Other(x),
}; };
info!( info!(
"checkpoint ran in {:?} (result: {:?}, WAL size: {})", "checkpoint ran in {:?} (result: {:?}, WAL size: {})",
@@ -131,7 +140,7 @@ pub fn checkpoint_db(conn: &mut PooledConnection) -> Result<()> {
result, result,
wal_size wal_size
); );
Ok(()) Ok(wal_size as usize)
} }
/// Spawn a database writer that persists events to the SQLite store. /// Spawn a database writer that persists events to the SQLite store.
@@ -399,7 +408,7 @@ pub fn write_event(conn: &mut PooledConnection, e: &Event) -> Result<usize> {
// if this event is replaceable update, hide every other replaceable // if this event is replaceable update, hide every other replaceable
// event with the same kind from the same author that was issued // event with the same kind from the same author that was issued
// earlier than this. // earlier than this.
if e.kind == 0 || e.kind == 3 || (e.kind >= 10000 && e.kind < 20000) { if e.kind == 0 || e.kind == 3 || e.kind == 41 || (e.kind >= 10000 && e.kind < 20000) {
let update_count = tx.execute( let update_count = tx.execute(
"UPDATE event SET hidden=TRUE WHERE id!=? AND kind=? AND author=? AND created_at <= ? and hidden!=TRUE", "UPDATE event SET hidden=TRUE WHERE id!=? AND kind=? AND author=? AND created_at <= ? and hidden!=TRUE",
params![ev_id, e.kind, hex::decode(&e.pubkey).ok(), e.created_at], params![ev_id, e.kind, hex::decode(&e.pubkey).ok(), e.created_at],
@@ -671,32 +680,63 @@ fn _pool_at_capacity(pool: &SqlitePool) -> bool {
fn log_pool_stats(name: &str, pool: &SqlitePool) { fn log_pool_stats(name: &str, pool: &SqlitePool) {
let state: r2d2::State = pool.state(); let state: r2d2::State = pool.state();
let in_use_cxns = state.connections - state.idle_connections; let in_use_cxns = state.connections - state.idle_connections;
trace!( debug!(
"DB pool {:?} usage (in_use: {}, available: {})", "DB pool {:?} usage (in_use: {}, available: {}, max: {})",
name, name,
in_use_cxns, in_use_cxns,
state.connections state.connections,
pool.max_size()
); );
if state.connections == in_use_cxns {
debug!("DB pool {:?} is empty (in_use: {})", name, in_use_cxns);
}
} }
/// Perform database maintenance on a regular basis /// Perform database maintenance on a regular basis
pub async fn db_maintenance(pool: SqlitePool) { pub async fn db_optimize(pool: SqlitePool) {
tokio::task::spawn(async move { tokio::task::spawn(async move {
loop { loop {
tokio::select! { tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(EVENT_MAINTENANCE_FREQ_SEC)) => { _ = tokio::time::sleep(Duration::from_secs(60*60)) => {
if let Ok(mut conn) = pool.get() { if let Ok(mut conn) = pool.get() {
// the busy timer will block writers, so don't set // the busy timer will block writers, so don't set
// this any higher than you want max latency for event // this any higher than you want max latency for event
// writes. // writes.
conn.busy_timeout(Duration::from_secs(1)).ok(); info!("running database optimizer");
debug!("running database optimizer");
optimize_db(&mut conn).ok(); optimize_db(&mut conn).ok();
}
}
};
}
});
}
/// Perform database WAL checkpoint on a regular basis
pub async fn db_checkpoint(pool: SqlitePool) {
tokio::task::spawn(async move {
// WAL size in pages.
let mut current_wal_size = 0;
// WAL threshold for more aggressive checkpointing (10,000 pages, or about 40MB)
let wal_threshold = 1000*10;
// default threshold for the busy timer
let busy_wait_default = Duration::from_secs(1);
// if the WAL file is getting too big, switch to this
let busy_wait_default_long = Duration::from_secs(5);
loop {
tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(CHECKPOINT_FREQ_SEC)) => {
if let Ok(mut conn) = pool.get() {
// the busy timer will block writers, so don't set
// this any higher than you want max latency for event
// writes.
if current_wal_size <= wal_threshold {
conn.busy_timeout(busy_wait_default).ok();
} else {
// if the wal size has exceeded a threshold, increase the busy timeout.
conn.busy_timeout(busy_wait_default_long).ok();
}
debug!("running wal_checkpoint(TRUNCATE)"); debug!("running wal_checkpoint(TRUNCATE)");
checkpoint_db(&mut conn).ok(); if let Ok(new_size) = checkpoint_db(&mut conn) {
current_wal_size = new_size;
}
} }
} }
}; };
@@ -743,8 +783,6 @@ pub async fn db_query(
if sql_gen_elapsed > Duration::from_millis(10) { if sql_gen_elapsed > Duration::from_millis(10) {
debug!("SQL (slow) generated in {:?}", start.elapsed()); debug!("SQL (slow) generated in {:?}", start.elapsed());
} }
// show pool stats
log_pool_stats("reader", &pool);
// cutoff for displaying slow queries // cutoff for displaying slow queries
let slow_cutoff = Duration::from_millis(2000); let slow_cutoff = Duration::from_millis(2000);
// any client that doesn't cause us to generate new rows in 5 // any client that doesn't cause us to generate new rows in 5
@@ -753,10 +791,13 @@ pub async fn db_query(
let start = Instant::now(); let start = Instant::now();
let mut slow_first_event; let mut slow_first_event;
let mut last_successful_send = Instant::now(); let mut last_successful_send = Instant::now();
if let Ok(conn) = pool.get() { if let Ok(mut conn) = pool.get() {
// execute the query. Don't cache, since queries vary so much. // execute the query.
let mut stmt = conn.prepare(&q)?; // make the actual SQL query (with parameters inserted) available
conn.trace(Some(|x| {trace!("SQL trace: {:?}", x)}));
let mut stmt = conn.prepare_cached(&q)?;
let mut event_rows = stmt.query(rusqlite::params_from_iter(p))?; let mut event_rows = stmt.query(rusqlite::params_from_iter(p))?;
let mut first_result = true; let mut first_result = true;
while let Some(row) = event_rows.next()? { while let Some(row) = event_rows.next()? {
let first_event_elapsed = start.elapsed(); let first_event_elapsed = start.elapsed();
@@ -770,28 +811,11 @@ pub async fn db_query(
} }
// logging for slow queries; show sub and SQL. // logging for slow queries; show sub and SQL.
// to reduce logging; only show 1/16th of clients (leading 0) // to reduce logging; only show 1/16th of clients (leading 0)
if row_count == 0 && slow_first_event && client_id.starts_with("0") { if row_count == 0 && slow_first_event && client_id.starts_with('0') {
debug!( debug!(
"query req (slow): {:?} (cid: {}, sub: {:?})", "query req (slow): {:?} (cid: {}, sub: {:?})",
sub, client_id, sub.id sub, client_id, sub.id
); );
debug!(
"query string (slow): {} (cid: {}, sub: {:?})",
q, client_id, sub.id
);
} else {
trace!(
"query req: {:?} (cid: {}, sub: {:?})",
sub,
client_id,
sub.id
);
trace!(
"query string: {} (cid: {}, sub: {:?})",
q,
client_id,
sub.id
);
} }
// check if this is still active; every 100 rows // check if this is still active; every 100 rows
if row_count % 100 == 0 && abandon_query_rx.try_recv().is_ok() { if row_count % 100 == 0 && abandon_query_rx.try_recv().is_ok() {
@@ -809,7 +833,8 @@ pub async fn db_query(
trace!("db reader thread is stalled"); trace!("db reader thread is stalled");
if last_successful_send + abort_cutoff < Instant::now() { if last_successful_send + abort_cutoff < Instant::now() {
// the queue has been full for too long, abort // the queue has been full for too long, abort
info!("aborting database query due to slow client"); info!("aborting database query due to slow client (cid: {}, sub: {:?})",
client_id, sub.id);
let ok: Result<()> = Ok(()); let ok: Result<()> = Ok(());
return ok; return ok;
} }
+7 -20
View File
@@ -80,7 +80,7 @@ impl FromStr for Operator {
#[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)] #[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)]
pub struct ConditionQuery { pub struct ConditionQuery {
pub(crate) conditions: Vec<Condition>, pub conditions: Vec<Condition>,
} }
impl ConditionQuery { impl ConditionQuery {
@@ -137,9 +137,9 @@ pub fn validate_delegation(
/// An example complex condition would be: kind=1,2,3&created_at<1665265999 /// An example complex condition would be: kind=1,2,3&created_at<1665265999
#[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)] #[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)]
pub struct Condition { pub struct Condition {
pub(crate) field: Field, pub field: Field,
pub(crate) operator: Operator, pub operator: Operator,
pub(crate) values: Vec<u64>, pub values: Vec<u64>,
} }
impl Condition { impl Condition {
@@ -332,19 +332,6 @@ mod tests {
assert_eq!(parsed, cq); assert_eq!(parsed, cq);
Ok(()) Ok(())
} }
fn simple_event() -> Event {
Event {
id: "0".to_owned(),
pubkey: "0".to_owned(),
delegated_by: None,
created_at: 0,
kind: 0,
tags: vec![],
content: "".to_owned(),
sig: "0".to_owned(),
tagidx: None,
}
}
// Check for condition logic on event w/ empty values // Check for condition logic on event w/ empty values
#[test] #[test]
fn condition_with_empty_values() { fn condition_with_empty_values() {
@@ -353,7 +340,7 @@ mod tests {
operator: Operator::GreaterThan, operator: Operator::GreaterThan,
values: vec![], values: vec![],
}; };
let e = simple_event(); let e = Event::simple_event();
assert!(!c.allows_event(&e)); assert!(!c.allows_event(&e));
c.operator = Operator::LessThan; c.operator = Operator::LessThan;
assert!(!c.allows_event(&e)); assert!(!c.allows_event(&e));
@@ -373,7 +360,7 @@ mod tests {
operator: Operator::GreaterThan, operator: Operator::GreaterThan,
values: vec![10], values: vec![10],
}; };
let mut e = simple_event(); let mut e = Event::simple_event();
// kind is not greater than 10, not allowed // kind is not greater than 10, not allowed
e.kind = 1; e.kind = 1;
assert!(!c.allows_event(&e)); assert!(!c.allows_event(&e));
@@ -392,7 +379,7 @@ mod tests {
operator: Operator::Equals, operator: Operator::Equals,
values: vec![0, 10, 20], values: vec![0, 10, 20],
}; };
let mut e = simple_event(); let mut e = Event::simple_event();
// Allow if event kind is in list for Equals // Allow if event kind is in list for Equals
e.kind = 10; e.kind = 10;
assert!(c.allows_event(&e)); assert!(c.allows_event(&e));
+29 -27
View File
@@ -37,19 +37,19 @@ impl EventCmd {
#[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)] #[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone)]
pub struct Event { pub struct Event {
pub id: String, pub id: String,
pub(crate) pubkey: String, pub pubkey: String,
#[serde(skip)] #[serde(skip)]
pub(crate) delegated_by: Option<String>, pub delegated_by: Option<String>,
pub(crate) created_at: u64, pub created_at: u64,
pub(crate) kind: u64, pub kind: u64,
#[serde(deserialize_with = "tag_from_string")] #[serde(deserialize_with = "tag_from_string")]
// NOTE: array-of-arrays may need to be more general than a string container // NOTE: array-of-arrays may need to be more general than a string container
pub(crate) tags: Vec<Vec<String>>, pub tags: Vec<Vec<String>>,
pub(crate) content: String, pub content: String,
pub(crate) sig: String, pub sig: String,
// Optimization for tag search, built on demand. // Optimization for tag search, built on demand.
#[serde(skip)] #[serde(skip)]
pub(crate) tagidx: Option<HashMap<char, HashSet<String>>>, pub tagidx: Option<HashMap<char, HashSet<String>>>,
} }
/// Simple tag type for array of array of strings. /// Simple tag type for array of array of strings.
@@ -101,6 +101,21 @@ impl From<EventCmd> for Result<Event> {
} }
impl Event { impl Event {
#[cfg(test)]
pub fn simple_event() -> Event {
Event {
id: "0".to_owned(),
pubkey: "0".to_owned(),
delegated_by: None,
created_at: 0,
kind: 0,
tags: vec![],
content: "".to_owned(),
sig: "0".to_owned(),
tagidx: None,
}
}
pub fn is_kind_metadata(&self) -> bool { pub fn is_kind_metadata(&self) -> bool {
self.kind == 0 self.kind == 0
} }
@@ -226,7 +241,7 @@ impl Event {
} }
/// Check if this event has a valid signature. /// Check if this event has a valid signature.
fn validate(&self) -> Result<()> { pub fn validate(&self) -> Result<()> {
// TODO: return a Result with a reason for invalid events // TODO: return a Result with a reason for invalid events
// validation is performed by: // validation is performed by:
// * parsing JSON string into event fields // * parsing JSON string into event fields
@@ -319,31 +334,18 @@ impl Event {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
fn simple_event() -> Event {
Event {
id: "0".to_owned(),
pubkey: "0".to_owned(),
delegated_by: None,
created_at: 0,
kind: 0,
tags: vec![],
content: "".to_owned(),
sig: "0".to_owned(),
tagidx: None,
}
}
#[test] #[test]
fn event_creation() { fn event_creation() {
// create an event // create an event
let event = simple_event(); let event = Event::simple_event();
assert_eq!(event.id, "0"); assert_eq!(event.id, "0");
} }
#[test] #[test]
fn event_serialize() -> Result<()> { fn event_serialize() -> Result<()> {
// serialize an event to JSON string // serialize an event to JSON string
let event = simple_event(); let event = Event::simple_event();
let j = serde_json::to_string(&event)?; let j = serde_json::to_string(&event)?;
assert_eq!(j, "{\"id\":\"0\",\"pubkey\":\"0\",\"created_at\":0,\"kind\":0,\"tags\":[],\"content\":\"\",\"sig\":\"0\"}"); assert_eq!(j, "{\"id\":\"0\",\"pubkey\":\"0\",\"created_at\":0,\"kind\":0,\"tags\":[],\"content\":\"\",\"sig\":\"0\"}");
Ok(()) Ok(())
@@ -351,14 +353,14 @@ mod tests {
#[test] #[test]
fn empty_event_tag_match() { fn empty_event_tag_match() {
let event = simple_event(); let event = Event::simple_event();
assert!(!event assert!(!event
.generic_tag_val_intersect('e', &HashSet::from(["foo".to_owned(), "bar".to_owned()]))); .generic_tag_val_intersect('e', &HashSet::from(["foo".to_owned(), "bar".to_owned()])));
} }
#[test] #[test]
fn single_event_tag_match() { fn single_event_tag_match() {
let mut event = simple_event(); let mut event = Event::simple_event();
event.tags = vec![vec!["e".to_owned(), "foo".to_owned()]]; event.tags = vec![vec!["e".to_owned(), "foo".to_owned()]];
event.build_index(); event.build_index();
assert_eq!( assert_eq!(
@@ -373,7 +375,7 @@ mod tests {
#[test] #[test]
fn event_tags_serialize() -> Result<()> { fn event_tags_serialize() -> Result<()> {
// serialize an event with tags to JSON string // serialize an event with tags to JSON string
let mut event = simple_event(); let mut event = Event::simple_event();
event.tags = vec![ event.tags = vec![
vec![ vec![
"e".to_owned(), "e".to_owned(),
+27 -8
View File
@@ -29,6 +29,7 @@ use std::collections::HashMap;
use std::convert::Infallible; use std::convert::Infallible;
use std::net::SocketAddr; use std::net::SocketAddr;
use std::path::Path; use std::path::Path;
use std::sync::atomic::Ordering;
use std::sync::mpsc::Receiver as MpscReceiver; use std::sync::mpsc::Receiver as MpscReceiver;
use std::time::Duration; use std::time::Duration;
use std::time::Instant; use std::time::Instant;
@@ -88,6 +89,7 @@ async fn handle_web_request(
Some(config), Some(config),
) )
.await; .await;
let origin = get_header_string("origin", request.headers());
let user_agent = get_header_string("user-agent", request.headers()); let user_agent = get_header_string("user-agent", request.headers());
// determine the remote IP from headers if the exist // determine the remote IP from headers if the exist
let header_ip = settings let header_ip = settings
@@ -101,6 +103,7 @@ async fn handle_web_request(
let client_info = ClientInfo { let client_info = ClientInfo {
remote_ip, remote_ip,
user_agent, user_agent,
origin,
}; };
// spawn a nostr server with our websocket // spawn a nostr server with our websocket
tokio::spawn(nostr_server( tokio::spawn(nostr_server(
@@ -245,14 +248,19 @@ pub fn start_server(settings: Settings, shutdown_rx: MpscReceiver<()>) -> Result
// configure tokio runtime // configure tokio runtime
let rt = Builder::new_multi_thread() let rt = Builder::new_multi_thread()
.enable_all() .enable_all()
.thread_name("tokio-ws") .thread_name_fn(|| {
// give each thread a unique numeric name
static ATOMIC_ID: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let id = ATOMIC_ID.fetch_add(1,Ordering::SeqCst);
format!("tokio-ws-{}", id)
})
// limit concurrent SQLite blocking threads // limit concurrent SQLite blocking threads
.max_blocking_threads(settings.limits.max_blocking_threads) .max_blocking_threads(settings.limits.max_blocking_threads)
.on_thread_start(|| { .on_thread_start(|| {
trace!("started new thread"); trace!("started new thread: {:?}", std::thread::current().name());
}) })
.on_thread_stop(|| { .on_thread_stop(|| {
trace!("stopping thread"); trace!("stopped thread: {:?}", std::thread::current().name());
}) })
.build() .build()
.unwrap(); .unwrap();
@@ -317,10 +325,11 @@ pub fn start_server(settings: Settings, shutdown_rx: MpscReceiver<()>) -> Result
&settings, &settings,
OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE,
1, 1,
1, 2,
false, false,
); );
db::db_maintenance(maintenance_pool).await; db::db_optimize(maintenance_pool.clone()).await;
db::db_checkpoint(maintenance_pool).await;
// listen for (external to tokio) shutdown request // listen for (external to tokio) shutdown request
let controlled_shutdown = invoke_shutdown.clone(); let controlled_shutdown = invoke_shutdown.clone();
@@ -356,6 +365,10 @@ pub fn start_server(settings: Settings, shutdown_rx: MpscReceiver<()>) -> Result
db_max_conn, db_max_conn,
true, true,
); );
// spawn a task to check the pool size.
let pool_monitor = pool.clone();
tokio::spawn(async move {db::monitor_pool("reader", pool_monitor).await;});
// A `Service` is needed for every connection, so this // A `Service` is needed for every connection, so this
// creates one from our `handle_request` function. // creates one from our `handle_request` function.
let make_svc = make_service_fn(|conn: &AddrStream| { let make_svc = make_service_fn(|conn: &AddrStream| {
@@ -439,6 +452,7 @@ fn make_notice_message(notice: Notice) -> Message {
struct ClientInfo { struct ClientInfo {
remote_ip: String, remote_ip: String,
user_agent: Option<String>, user_agent: Option<String>,
origin: Option<String>,
} }
/// Handle new client connections. This runs through an event loop /// Handle new client connections. This runs through an event loop
@@ -502,9 +516,14 @@ async fn nostr_server(
let mut client_published_event_count: usize = 0; let mut client_published_event_count: usize = 0;
let mut client_received_event_count: usize = 0; let mut client_received_event_count: usize = 0;
debug!("new client connection (cid: {}, ip: {:?})", cid, conn.ip()); debug!("new client connection (cid: {}, ip: {:?})", cid, conn.ip());
if let Some(ua) = client_info.user_agent { let origin = client_info.origin.unwrap_or_else(|| "<unspecified>".into());
debug!("cid: {}, user-agent: {:?}", cid, ua); let user_agent = client_info
} .user_agent
.unwrap_or_else(|| "<unspecified>".into());
debug!(
"cid: {}, origin: {:?}, user-agent: {:?}",
cid, origin, user_agent
);
loop { loop {
tokio::select! { tokio::select! {
_ = shutdown.recv() => { _ = shutdown.recv() => {
+37 -2
View File
@@ -65,12 +65,21 @@ impl<'de> Deserialize<'de> for ReqFilter {
tags: None, tags: None,
force_no_match: false, force_no_match: false,
}; };
let empty_string = "".into();
let mut ts = None; let mut ts = None;
// iterate through each key, and assign values that exist // iterate through each key, and assign values that exist
for (key, val) in filter.into_iter() { for (key, val) in filter.into_iter() {
// ids // ids
if key == "ids" { if key == "ids" {
rf.ids = Deserialize::deserialize(val).ok(); let raw_ids: Option<Vec<String>>= Deserialize::deserialize(val).ok();
if let Some(a) = raw_ids.as_ref() {
if a.contains(&empty_string) {
return Err(serde::de::Error::invalid_type(
Unexpected::Other("prefix matches must not be empty strings"),
&"a json object"));
}
}
rf.ids =raw_ids;
} else if key == "kinds" { } else if key == "kinds" {
rf.kinds = Deserialize::deserialize(val).ok(); rf.kinds = Deserialize::deserialize(val).ok();
} else if key == "since" { } else if key == "since" {
@@ -80,7 +89,15 @@ impl<'de> Deserialize<'de> for ReqFilter {
} else if key == "limit" { } else if key == "limit" {
rf.limit = Deserialize::deserialize(val).ok(); rf.limit = Deserialize::deserialize(val).ok();
} else if key == "authors" { } else if key == "authors" {
rf.authors = Deserialize::deserialize(val).ok(); let raw_authors: Option<Vec<String>>= Deserialize::deserialize(val).ok();
if let Some(a) = raw_authors.as_ref() {
if a.contains(&empty_string) {
return Err(serde::de::Error::invalid_type(
Unexpected::Other("prefix matches must not be empty strings"),
&"a json object"));
}
}
rf.authors = raw_authors;
} else if key.starts_with('#') && key.len() > 1 && val.is_array() { } else if key.starts_with('#') && key.len() > 1 && val.is_array() {
if let Some(tag_search) = tag_search_char_from_filter(key) { if let Some(tag_search) = tag_search_char_from_filter(key) {
if ts.is_none() { if ts.is_none() {
@@ -294,6 +311,24 @@ mod tests {
assert!(serde_json::from_str::<Subscription>(raw_json).is_err()); assert!(serde_json::from_str::<Subscription>(raw_json).is_err());
} }
#[test]
fn req_empty_authors_prefix() {
let raw_json = "[\"REQ\",\"some-id\",{\"authors\": [\"\"]}]";
assert!(serde_json::from_str::<Subscription>(raw_json).is_err());
}
#[test]
fn req_empty_ids_prefix() {
let raw_json = "[\"REQ\",\"some-id\",{\"ids\": [\"\"]}]";
assert!(serde_json::from_str::<Subscription>(raw_json).is_err());
}
#[test]
fn req_empty_ids_prefix_mixed() {
let raw_json = "[\"REQ\",\"some-id\",{\"ids\": [\"\",\"aaa\"]}]";
assert!(serde_json::from_str::<Subscription>(raw_json).is_err());
}
#[test] #[test]
fn legacy_filter() { fn legacy_filter() {
// legacy field in filter // legacy field in filter