build: bump version to 0.7.16

Updating anyhow v1.0.67 -> v1.0.68
Updating cxx v1.0.84 -> v1.0.85
Updating cxx-build v1.0.84 -> v1.0.85
Updating cxxbridge-flags v1.0.84 -> v1.0.85
Updating cxxbridge-macro v1.0.84 -> v1.0.85
Updating hermit-abi v0.1.19 -> v0.2.6
Updating libc v0.2.138 -> v0.2.139
Updating nom v7.1.1 -> v7.1.2
Updating num_cpus v1.14.0 -> v1.15.0
Updating once_cell v1.16.0 -> v1.17.0
Updating openssl v0.10.44 -> v0.10.45
Updating openssl-sys v0.9.79 -> v0.9.80
Updating pest v2.5.1 -> v2.5.2
Updating pest_derive v2.5.1 -> v2.5.2
Updating pest_generator v2.5.1 -> v2.5.2
Updating pest_meta v2.5.1 -> v2.5.2
Updating proc-macro2 v1.0.48 -> v1.0.49
Updating prost v0.11.3 -> v0.11.5
Updating prost-derive v0.11.2 -> v0.11.5
Updating prost-types v0.11.2 -> v0.11.5
Updating quote v1.0.22 -> v1.0.23
Updating serde v1.0.151 -> v1.0.152
Updating serde_derive v1.0.151 -> v1.0.152
Updating serde_json v1.0.90 -> v1.0.91
Updating syn v1.0.106 -> v1.0.107
Updating tokio v1.23.0 -> v1.23.1

Cargo.lock

@@ -54,9 +54,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.67"
+version = "1.0.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7724808837b77f4b4de9d283820f9d98bcf496d5692934b857a2399d31ff22e6"
+checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
 
 [[package]]
 name = "async-stream"
@@ -382,9 +382,9 @@ dependencies = [
 
 [[package]]
 name = "cxx"
-version = "1.0.84"
+version = "1.0.85"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27874566aca772cb515af4c6e997b5fe2119820bca447689145e39bb734d19a0"
+checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd"
 dependencies = [
  "cc",
  "cxxbridge-flags",
@@ -394,9 +394,9 @@ dependencies = [
 
 [[package]]
 name = "cxx-build"
-version = "1.0.84"
+version = "1.0.85"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7bb951f2523a49533003656a72121306b225ec16a49a09dc6b0ba0d6f3ec3c0"
+checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0"
 dependencies = [
  "cc",
  "codespan-reporting",
@@ -409,15 +409,15 @@ dependencies = [
 
 [[package]]
 name = "cxxbridge-flags"
-version = "1.0.84"
+version = "1.0.85"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be778b6327031c1c7b61dd2e48124eee5361e6aa76b8de93692f011b08870ab4"
+checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59"
 
 [[package]]
 name = "cxxbridge-macro"
-version = "1.0.84"
+version = "1.0.85"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b8a2b87662fe5a0a0b38507756ab66aff32638876a0866e5a5fc82ceb07ee49"
+checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -729,9 +729,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.19"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
 dependencies = [
  "libc",
 ]
@@ -933,9 +933,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.138"
+version = "0.2.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
+checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
 
 [[package]]
 name = "libsqlite3-sys"
@@ -1080,9 +1080,9 @@ checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
 
 [[package]]
 name = "nom"
-version = "7.1.1"
+version = "7.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
+checksum = "e5507769c4919c998e69e49c839d9dc6e693ede4cc4290d6ad8b41d4f09c548c"
 dependencies = [
  "memchr",
  "minimal-lexical",
@@ -1096,7 +1096,7 @@ checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
 
 [[package]]
 name = "nostr-rs-relay"
-version = "0.7.14"
+version = "0.7.16"
 dependencies = [
  "anyhow",
  "bitcoin_hashes",
@@ -1209,9 +1209,9 @@ dependencies = [
 
 [[package]]
 name = "num_cpus"
-version = "1.14.0"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
 dependencies = [
  "hermit-abi",
  "libc",
@@ -1219,15 +1219,15 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.16.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
+checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
 
 [[package]]
 name = "openssl"
-version = "0.10.44"
+version = "0.10.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29d971fd5722fec23977260f6e81aa67d2f22cadbdc2aa049f1022d9a3be1566"
+checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1"
 dependencies = [
  "bitflags",
  "cfg-if",
@@ -1257,9 +1257,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.79"
+version = "0.9.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5454462c0eced1e97f2ec09036abc8da362e66802f66fd20f86854d9d8cbcbc4"
+checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7"
 dependencies = [
  "autocfg 1.1.0",
  "cc",
@@ -1326,9 +1326,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
 
 [[package]]
 name = "pest"
-version = "2.5.1"
+version = "2.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc8bed3549e0f9b0a2a78bf7c0018237a2cdf085eecbbc048e52612438e4e9d0"
+checksum = "0f6e86fb9e7026527a0d46bc308b841d73170ef8f443e1807f6ef88526a816d4"
 dependencies = [
  "thiserror",
  "ucd-trie",
@@ -1336,9 +1336,9 @@ dependencies = [
 
 [[package]]
 name = "pest_derive"
-version = "2.5.1"
+version = "2.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdc078600d06ff90d4ed238f0119d84ab5d43dbaad278b0e33a8820293b32344"
+checksum = "96504449aa860c8dcde14f9fba5c58dc6658688ca1fe363589d6327b8662c603"
 dependencies = [
  "pest",
  "pest_generator",
@@ -1346,9 +1346,9 @@ dependencies = [
 
 [[package]]
 name = "pest_generator"
-version = "2.5.1"
+version = "2.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28a1af60b1c4148bb269006a750cff8e2ea36aff34d2d96cf7be0b14d1bed23c"
+checksum = "798e0220d1111ae63d66cb66a5dcb3fc2d986d520b98e49e1852bfdb11d7c5e7"
 dependencies = [
  "pest",
  "pest_meta",
@@ -1359,9 +1359,9 @@ dependencies = [
 
 [[package]]
 name = "pest_meta"
-version = "2.5.1"
+version = "2.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fec8605d59fc2ae0c6c1aefc0c7c7a9769732017c0ce07f7a9cfffa7b4404f20"
+checksum = "984298b75898e30a843e278a9f2452c31e349a073a0ce6fd950a12a74464e065"
 dependencies = [
  "once_cell",
  "pest",
@@ -1414,18 +1414,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.48"
+version = "1.0.49"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9d89e5dba24725ae5678020bf8f1357a9aa7ff10736b551adbcd3f8d17d766f"
+checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "prost"
-version = "0.11.3"
+version = "0.11.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0b18e655c21ff5ac2084a5ad0611e827b3f92badf79f4910b5a5c58f4d87ff0"
+checksum = "c01db6702aa05baa3f57dec92b8eeeeb4cb19e894e73996b32a4093289e54592"
 dependencies = [
  "bytes",
  "prost-derive",
@@ -1433,9 +1433,9 @@ dependencies = [
 
 [[package]]
 name = "prost-derive"
-version = "0.11.2"
+version = "0.11.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "164ae68b6587001ca506d3bf7f1000bfa248d0e1217b618108fba4ec1d0cc306"
+checksum = "c8842bad1a5419bca14eac663ba798f6bc19c413c2fdceb5f3ba3b0932d96720"
 dependencies = [
  "anyhow",
  "itertools",
@@ -1446,9 +1446,9 @@ dependencies = [
 
 [[package]]
 name = "prost-types"
-version = "0.11.2"
+version = "0.11.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "747761bc3dc48f9a34553bf65605cf6cb6288ba219f3450b4275dbd81539551a"
+checksum = "017f79637768cde62820bc2d4fe0e45daaa027755c323ad077767c6c5f173091"
 dependencies = [
  "bytes",
  "prost",
@@ -1472,9 +1472,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.22"
+version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "556d0f47a940e895261e77dc200d5eadfc6ef644c179c6f5edfc105e3a2292c8"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
 dependencies = [
  "proc-macro2",
 ]
@@ -1823,18 +1823,18 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.151"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0"
+checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.151"
+version = "1.0.152"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "255abe9a125a985c05190d687b320c12f9b1f0b99445e608c21ba0782c719ad8"
+checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1843,9 +1843,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.90"
+version = "1.0.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8778cc0b528968fe72abec38b5db5a20a70d148116cd9325d2bc5f5180ca3faf"
+checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
 dependencies = [
  "indexmap",
  "itoa",
@@ -1920,9 +1920,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.106"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09ee3a69cd2c7e06684677e5629b3878b253af05e4714964204279c6bc02cf0b"
+checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2004,9 +2004,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.23.0"
+version = "1.23.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46"
+checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8"
 dependencies = [
  "autocfg 1.1.0",
  "bytes",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nostr-rs-relay"
-version = "0.7.14"
+version = "0.7.16"
 edition = "2021"
 authors = ["Greg Heartsfield <scsibug@imap.cc>"]
 description = "A relay implementation for the Nostr protocol"

README.md

@@ -28,7 +28,7 @@ mirrored on [GitHub](https://github.com/scsibug/nostr-rs-relay).
 - [x] NIP-16: [Event Treatment](https://github.com/nostr-protocol/nips/blob/master/16.md)
 - [x] NIP-20: [Command Results](https://github.com/nostr-protocol/nips/blob/master/20.md)
 - [x] NIP-22: [Event `created_at` limits](https://github.com/nostr-protocol/nips/blob/master/22.md) (_future-dated events only_)
-- [x] NIP-26: [Event Delegation](https://github.com/nostr-protocol/nips/blob/master/26.md)
+- [ ] NIP-26: [Event Delegation](https://github.com/nostr-protocol/nips/blob/master/26.md) (_implemented, but currently disabled_)
 - [x] NIP-28: [Public Chat](https://github.com/nostr-protocol/nips/blob/master/28.md)
 
 ## Quick Start

config.toml

@@ -100,6 +100,11 @@ reject_future_seconds = 1800
 # backpressure to senders if writes are slow.
 #event_persist_buffer = 4096
 
+# Event kind blacklist. Events with these kinds will be discarded.
+#event_kind_blacklist = [
+#    70202,
+#]
+
 [authorization]
 # Pubkey addresses in this array are whitelisted for event publishing.
 # Only valid events by these authors will be accepted, if the variable

docs/database-maintenance.md

@@ -0,0 +1,125 @@
+# Database Maintenance
+
+`nostr-rs-relay` uses the SQLite embedded database to minimize
+dependencies and overall footprint of running a relay.  If traffic is
+light, the relay should just run with very little need for
+intervention.  For heavily trafficked relays, there are a number of
+steps that the operator may need to take to maintain performance and
+limit disk usage.
+
+This maintenance guide is current as of version `0.7.14`.  Future
+versions may incorporate and automate some of these steps.
+
+## Backing Up the Database
+
+To prevent data loss, the database should be backed up regularly.  The
+recommended method is to use the `sqlite3` command to perform an
+"Online Backup".  This can be done while the relay is running, queries
+can still run and events will be persisted during the backup.
+
+The following commands will perform a backup of the database to a
+dated file, and then compress to minimize size:
+
+```console
+BACKUP_FILE=/var/backups/nostr/`date +%Y%m%d_%H%M`.db
+sqlite3 -readonly /apps/nostr-relay/nostr.db ".backup $BACKUP_FILE
+sqlite3 $BACKUP_FILE "vacuum;"
+bzip2 -9 $BACKUP_FILE
+```
+
+Nostr events are very compressible.  Expect a compression ratio on the
+order of 4:1, resulting in a 75% space saving.
+
+## Vacuuming the Database
+
+As the database is updated, it can become fragmented.  Performing a
+full `vacuum` will rebuild the entire database file, and can reduce
+space.  Running this may reduce the size of the database file,
+especially if a large amount of data was updated or deleted.
+
+```console
+vacuum;
+```
+
+## Clearing Hidden Events
+
+When events are deleted, either through deletion events, metadata or
+follower updates, or a replaceable event kind, the event is not
+actually removed from the database.  Instead, a flag `HIDDEN` is set
+to true for the event, which excludes it from search results.  The
+original intent was to ensure that subsequent rebroadcasts of the
+event would be easily detected as having been deleted, and would not
+need to be stored again.  In practice, this decision causes excessive
+growth of the `tags` table, since all the previous followers are
+retained for those `HIDDEN` events.
+
+The `event` and especially the `tag` table can be significantly
+reduced in size by running these commands:
+
+```console
+PRAGMA foreign_keys = ON;
+delete from event where HIDDEN=true;
+```
+
+## Manually Removing Events
+
+For a variety of reasons, an operator may wish to remove some events
+from the database.  The only way of achieving this today is with
+manually run SQL commands.
+
+It is recommended to have a good backup prior to manually running SQL
+commands!
+
+In all cases, it is mandatory to enable foreign keys, and this must be
+done for every connection.  Otherwise, you will likely orphan rows in
+the `tag` table.
+
+### Deleting Specific Event
+
+```console
+PRAGMA foreign_keys = ON;
+delete from event where event_hash=x'00000000000c1271675dc86e3e1dd1336827bccabb90dc4c9d3b4465efefe00e';
+```
+
+### Deleting All Events for Pubkey
+
+```console
+PRAGMA foreign_keys = ON;
+delete from event where author=x'000000000002c7831d9c5a99f183afc2813a6f69a16edda7f6fc0ed8110566e6';
+```
+
+### Deleting All Events of a Kind
+
+
+```console
+PRAGMA foreign_keys = ON;
+delete from event where kind=70202;
+```
+
+### Deleting Old Events
+
+In this scenario, we wish to delete any event that has been stored by
+our relay for more than 1 month.  Crucially, this is based on when the
+event was stored, not when the event says it was created.  If an event
+has a `created` field of 2 years ago, but was first sent to our relay
+yesterday, it would not be deleted in this scenario.  Keep in mind, we
+do not track anything for re-broadcast events that we already have, so
+this is not a very effective way of implementing a "least recently
+seen" policy.
+
+```console
+PRAGMA foreign_keys = ON;
+TODO!
+```
+
+### Delete Profile Events with No Recent Events
+
+Many users create profiles, post a "hello world" event, and then never
+appear again (likely using an ephemeral keypair that was lost in the
+browser cache).  We can find these accounts and remove them after some
+time.
+
+```console
+PRAGMA foreign_keys = ON;
+TODO!
+```

src/config.rs

@@ -60,6 +60,7 @@ pub struct Limits {
     pub max_ws_frame_bytes: Option<usize>,
     pub broadcast_buffer: usize, // events to buffer for subscribers (prevents slow readers from consuming memory)
     pub event_persist_buffer: usize, // events to buffer for database commits (block senders if database writes are too slow)
+    pub event_kind_blacklist: Option<Vec<u64>>
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -225,6 +226,7 @@ impl Default for Settings {
                 max_ws_frame_bytes: Some(2 << 17),   // 128K
                 broadcast_buffer: 16384,
                 event_persist_buffer: 4096,
+                event_kind_blacklist: None,
             },
             authorization: Authorization {
                 pubkey_whitelist: None, // Allow any address to publish

src/db.rs

@@ -19,8 +19,10 @@ use r2d2_sqlite::SqliteConnectionManager;
 use rusqlite::params;
 use rusqlite::types::ToSql;
 use rusqlite::OpenFlags;
+use tokio::sync::{Mutex, MutexGuard};
 use std::fmt::Write as _;
 use std::path::Path;
+use std::sync::Arc;
 use std::thread;
 use std::time::Duration;
 use std::time::Instant;
@@ -42,10 +44,6 @@ pub const DB_FILE: &str = "nostr.db";
 /// How frequently to attempt checkpointing
 pub const CHECKPOINT_FREQ_SEC: u64 = 60;
 
-/// How many persisted events before we pause for backups.
-/// It isn't clear this is enough to make the online backup API work yet.
-pub const EVENT_COUNT_BACKUP_PAUSE_TRIGGER: usize = 1000;
-
 /// Build a database connection pool.
 /// # Panics
 ///
@@ -182,10 +180,6 @@ pub async fn db_writer(
         let rps_setting = settings.limits.messages_per_sec;
         let mut most_recent_rate_limit = Instant::now();
         let mut lim_opt = None;
-        // Constant writing has interfered with online backups.  Keep
-        // track of how long since we've given the backups a chance to
-        // run.
-        let mut backup_pause_counter: usize = 0;
         let clock = governor::clock::QuantaClock::default();
         if let Some(rps) = rps_setting {
             if rps > 0 {
@@ -230,6 +224,24 @@ pub async fn db_writer(
                 }
             }
 
+            // Check that event kind isn't blacklisted
+            let kinds_blacklist = &settings.limits.event_kind_blacklist.clone();
+            if let Some(event_kind_blacklist) = kinds_blacklist {
+                if event_kind_blacklist.contains(&event.kind) {
+                    info!(
+                        "Rejecting event {}, blacklisted kind",
+                        &event.get_event_id_prefix()
+                    );
+                    notice_tx
+                        .try_send(Notice::blocked(
+                            event.id,
+                            "event kind is blocked by relay"
+                        ))
+                        .ok();
+                    continue;
+                }
+            }
+
             // send any metadata events to the NIP-05 verifier
             if nip05_active && event.is_kind_metadata() {
                 // we are sending this prior to even deciding if we
@@ -320,12 +332,6 @@ pub async fn db_writer(
                         notice_tx.try_send(Notice::error(event.id, msg)).ok();
                     }
                 }
-                backup_pause_counter += 1;
-                if backup_pause_counter > EVENT_COUNT_BACKUP_PAUSE_TRIGGER {
-                    info!("pausing db write thread for a moment...");
-                    thread::sleep(Duration::from_millis(500));
-                    backup_pause_counter = 0
-                }
             }
 
             // use rate limit, if defined, and if an event was actually written.
@@ -358,6 +364,9 @@ pub async fn db_writer(
 
 /// Persist an event to the database, returning rows added.
 pub fn write_event(conn: &mut PooledConnection, e: &Event) -> Result<usize> {
+    // enable auto vacuum
+    conn.execute_batch("pragma auto_vacuum = FULL")?;
+
     // start transaction
     let tx = conn.transaction()?;
     // get relevant fields from event and convert to blobs.
@@ -409,9 +418,11 @@ pub fn write_event(conn: &mut PooledConnection, e: &Event) -> Result<usize> {
     // event with the same kind from the same author that was issued
     // earlier than this.
     if e.kind == 0 || e.kind == 3 || e.kind == 41 || (e.kind >= 10000 && e.kind < 20000) {
+	let author = hex::decode(&e.pubkey).ok();
+	// this is a backwards check - hide any events that were older.
         let update_count = tx.execute(
-            "UPDATE event SET hidden=TRUE WHERE id!=? AND kind=? AND author=? AND created_at <= ? and hidden!=TRUE",
+            "UPDATE event SET hidden=TRUE WHERE hidden!=TRUE and kind=? and author=? and id NOT IN (SELECT id FROM event WHERE kind=? AND author=? ORDER BY created_at DESC LIMIT 1)",
-            params![ev_id, e.kind, hex::decode(&e.pubkey).ok(), e.created_at],
+            params![e.kind, author, e.kind, author],
         )?;
         if update_count > 0 {
             info!(
@@ -518,22 +529,18 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>) {
         for auth in authvec {
             match hex_range(auth) {
                 Some(HexSearch::Exact(ex)) => {
-                    auth_searches.push("author=? OR delegated_by=?".to_owned());
+                    auth_searches.push("author=?".to_owned());
-                    params.push(Box::new(ex.clone()));
                     params.push(Box::new(ex));
                 }
                 Some(HexSearch::Range(lower, upper)) => {
                     auth_searches.push(
-                        "(author>? AND author<?) OR (delegated_by>? AND delegated_by<?)".to_owned(),
+                        "(author>? AND author<?)".to_owned(),
                     );
-                    params.push(Box::new(lower.clone()));
-                    params.push(Box::new(upper.clone()));
                     params.push(Box::new(lower));
                     params.push(Box::new(upper));
                 }
                 Some(HexSearch::LowerOnly(lower)) => {
-                    auth_searches.push("author>? OR delegated_by>?".to_owned());
+                    auth_searches.push("author>?".to_owned());
-                    params.push(Box::new(lower.clone()));
                     params.push(Box::new(lower));
                 }
                 None => {
@@ -542,11 +549,9 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>) {
             }
         }
         if !authvec.is_empty() {
-            let authors_clause = format!("({})", auth_searches.join(" OR "));
+	    let auth_clause = format!("({})", auth_searches.join(" OR "));
-            filter_components.push(authors_clause);
+            filter_components.push(auth_clause);
         } else {
-            // if the authors list was empty, we should never return
-            // any results.
 	    filter_components.push("false".to_owned());
 	}
     }
@@ -691,7 +696,7 @@ fn log_pool_stats(name: &str, pool: &SqlitePool) {
 
 
 /// Perform database maintenance on a regular basis
-pub async fn db_optimize(pool: SqlitePool) {
+pub async fn db_optimize_task(pool: SqlitePool) {
     tokio::task::spawn(async move {
         loop {
             tokio::select! {
@@ -710,7 +715,7 @@ pub async fn db_optimize(pool: SqlitePool) {
 }
 
 /// Perform database WAL checkpoint on a regular basis
-pub async fn db_checkpoint(pool: SqlitePool) {
+pub async fn db_checkpoint_task(pool: SqlitePool, safe_to_read: Arc<Mutex<u64>>) {
     tokio::task::spawn(async move {
 	// WAL size in pages.
 	let mut current_wal_size = 0;
@@ -719,11 +724,12 @@ pub async fn db_checkpoint(pool: SqlitePool) {
 	// default threshold for the busy timer
 	let busy_wait_default = Duration::from_secs(1);
 	// if the WAL file is getting too big, switch to this
-	let busy_wait_default_long = Duration::from_secs(5);
+	let busy_wait_default_long = Duration::from_secs(10);
         loop {
             tokio::select! {
                 _ = tokio::time::sleep(Duration::from_secs(CHECKPOINT_FREQ_SEC)) => {
                     if let Ok(mut conn) = pool.get() {
+                        let mut _guard:Option<MutexGuard<u64>> = None;
 			// the busy timer will block writers, so don't set
 			// this any higher than you want max latency for event
 			// writes.
@@ -732,6 +738,9 @@ pub async fn db_checkpoint(pool: SqlitePool) {
 			} else {
 			    // if the wal size has exceeded a threshold, increase the busy timeout.
 			    conn.busy_timeout(busy_wait_default_long).ok();
+			    // take a lock that will prevent new readers.
+			    info!("blocking new readers to perform wal_checkpoint");
+			    _guard = Some(safe_to_read.lock().await);
 			}
                         debug!("running wal_checkpoint(TRUNCATE)");
                         if let Ok(new_size) = checkpoint_db(&mut conn) {
@@ -756,14 +765,19 @@ pub async fn db_query(
     pool: SqlitePool,
     query_tx: tokio::sync::mpsc::Sender<QueryResult>,
     mut abandon_query_rx: tokio::sync::oneshot::Receiver<()>,
+    safe_to_read: Arc<Mutex<u64>>,
 ) {
     let pre_spawn_start = Instant::now();
     task::spawn_blocking(move || {
+	{
+	    // if we are waiting on a checkpoint, stop until it is complete
+	    let _ = safe_to_read.blocking_lock();
+	}
         let db_queue_time = pre_spawn_start.elapsed();
         // if the queue time was very long (>5 seconds), spare the DB and abort.
         if db_queue_time > Duration::from_secs(5) {
             info!(
-                "shedding DB query load from {:?} (cid: {}, sub: {:?})",
+                "shedding DB query load queued for {:?} (cid: {}, sub: {:?})",
                 db_queue_time, client_id, sub.id
             );
             return Ok(());
@@ -817,6 +831,17 @@ pub async fn db_query(
                         sub, client_id, sub.id
                     );
 		}
+		// check if a checkpoint is trying to run, and abort
+		if row_count % 100 == 0 {
+		    {
+			if let Err(_) = safe_to_read.try_lock() {
+			    // lock was held, abort this query
+			    debug!("query aborted due to checkpoint (cid: {}, sub: {:?})", client_id, sub.id);
+			    return Ok(());
+			}
+		    }
+		}
+
                 // check if this is still active; every 100 rows
                 if row_count % 100 == 0 && abandon_query_rx.try_recv().is_ok() {
                     debug!("query aborted (cid: {}, sub: {:?})", client_id, sub.id);
@@ -838,6 +863,12 @@ pub async fn db_query(
                             let ok: Result<()> = Ok(());
                             return ok;
                         }
+			// check if a checkpoint is trying to run, and abort
+			if let Err(_) = safe_to_read.try_lock() {
+			    // lock was held, abort this query
+			    debug!("query aborted due to checkpoint (cid: {}, sub: {:?})", client_id, sub.id);
+			    return Ok(());
+			}
                         // give the queue a chance to clear before trying again
                         thread::sleep(Duration::from_millis(100));
                     }

src/info.rs

@@ -35,7 +35,7 @@ impl From<config::Info> for RelayInfo {
             description: i.description,
             pubkey: i.pubkey,
             contact: i.contact,
-            supported_nips: Some(vec![1, 2, 9, 11, 12, 15, 16, 20, 22, 26]),
+            supported_nips: Some(vec![1, 2, 9, 11, 12, 15, 16, 20, 22]),
             software: Some("https://git.sr.ht/~gheartsfield/nostr-rs-relay".to_owned()),
             version: CARGO_PKG_VERSION.map(|x| x.to_owned()),
         }

src/main.rs

@@ -18,6 +18,18 @@ fn db_from_args(args: &[String]) -> Option<String> {
     None
 }
 
+fn print_version() {
+    println!("{} v{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
+}
+
+fn print_help() {
+    println!("Usage: nostr-rs-relay [OPTION]...\n");
+    println!("Options:");
+    println!("  --help              Show this help message and exit");
+    println!("  --version           Show version information and exit");
+    println!("  --db <directory>    Use the <directory> as the location of the database");
+}
+
 /// Start running a Nostr relay server.
 fn main() {
     // setup tracing
@@ -25,6 +37,21 @@ fn main() {
     info!("Starting up from main");
     // get database directory from args
     let args: Vec<String> = env::args().collect();
+
+    let help_flag: bool = args.contains(&"--help".to_owned());
+    // if --help flag was passed, display help and exit
+    if help_flag {
+        print_help();
+        return;
+    }
+
+    let version_flag: bool = args.contains(&"--version".to_owned());
+    // if --version flag was passed, display version and exit
+    if version_flag {
+        print_version();
+        return;
+    }
+
     let db_dir: Option<String> = db_from_args(&args);
     // configure settings from config.toml
     // replace default settings with those read from config.toml

src/schema.rs

@@ -13,21 +13,22 @@ use tracing::{debug, error, info};
 
 /// Startup DB Pragmas
 pub const STARTUP_SQL: &str = r##"
-PRAGMA main.synchronous=NORMAL;
+PRAGMA main.synchronous = NORMAL;
 PRAGMA foreign_keys = ON;
-PRAGMA journal_size_limit=32768;
+PRAGMA journal_size_limit = 32768;
 pragma mmap_size = 17179869184; -- cap mmap at 16GB
 "##;
 
 /// Latest database version
-pub const DB_VERSION: usize = 11;
+pub const DB_VERSION: usize = 13;
 
 /// Schema definition
 const INIT_SQL: &str = formatcp!(
     r##"
 -- Database settings
 PRAGMA encoding = "UTF-8";
-PRAGMA journal_mode=WAL;
+PRAGMA journal_mode = WAL;
+PRAGMA auto_vacuum = FULL;
 PRAGMA main.synchronous=NORMAL;
 PRAGMA foreign_keys = ON;
 PRAGMA application_id = 1654008667;
@@ -52,6 +53,7 @@ CREATE INDEX IF NOT EXISTS author_index ON event(author);
 CREATE INDEX IF NOT EXISTS created_at_index ON event(created_at);
 CREATE INDEX IF NOT EXISTS delegated_by_index ON event(delegated_by);
 CREATE INDEX IF NOT EXISTS event_composite_index ON event(kind,created_at);
+CREATE INDEX IF NOT EXISTS kind_author_index ON event(kind,author);
 
 -- Tag Table
 -- Tag values are stored as either a BLOB (if they come in as a
@@ -171,6 +173,12 @@ pub fn upgrade_db(conn: &mut PooledConnection) -> Result<()> {
             if curr_version == 10 {
                 curr_version = mig_10_to_11(conn)?;
             }
+            if curr_version == 11 {
+                curr_version = mig_11_to_12(conn)?;
+            }
+            if curr_version == 12 {
+                curr_version = mig_12_to_13(conn)?;
+            }
 
             if curr_version == DB_VERSION {
                 info!(
@@ -371,7 +379,6 @@ fn mig_5_to_6(conn: &mut PooledConnection) -> Result<usize> {
 
 fn mig_6_to_7(conn: &mut PooledConnection) -> Result<usize> {
     info!("database schema needs update from 6->7");
-    // only change is adding a hidden column to events.
     let upgrade_sql = r##"
 ALTER TABLE event ADD delegated_by BLOB;
 CREATE INDEX IF NOT EXISTS delegated_by_index ON event(delegated_by);
@@ -469,3 +476,51 @@ PRAGMA user_version = 11;
     }
     Ok(11)
 }
+
+fn mig_11_to_12(conn: &mut PooledConnection) -> Result<usize> {
+    info!("database schema needs update from 11->12");
+    let start = Instant::now();
+    let tx = conn.transaction()?;
+    {
+        // Lookup every replaceable event
+        let mut stmt = tx.prepare("select kind,author from event where kind in (0,3,41) or (kind>=10000 and kind<20000) order by id;")?;
+        let mut replaceable_rows = stmt.query([])?;
+        while let Some(row) = replaceable_rows.next()? {
+            // we want to capture the event_id that had the tag, the tag name, and the tag hex value.
+            let event_kind: u64 = row.get(0)?;
+            let event_author: Vec<u8> = row.get(1)?;
+            tx.execute(
+                "UPDATE event SET hidden=TRUE WHERE hidden!=TRUE and kind=? and author=? and id NOT IN (SELECT id FROM event WHERE kind=? AND author=? ORDER BY created_at DESC LIMIT 1)",
+                params![event_kind, event_author, event_kind, event_author],
+            )?;
+        }
+        tx.execute("PRAGMA user_version = 12;", [])?;
+    }
+    tx.commit()?;
+    info!("database schema upgraded v11 -> v12 in {:?}", start.elapsed());
+    // vacuum after large table modification
+    let start = Instant::now();
+    conn.execute("VACUUM;", [])?;
+    info!("vacuumed DB after hidden event cleanup in {:?}", start.elapsed());
+    Ok(12)
+}
+
+fn mig_12_to_13(conn: &mut PooledConnection) -> Result<usize> {
+    info!("database schema needs update from 12->13");
+    let upgrade_sql = r##"
+CREATE INDEX IF NOT EXISTS kind_author_index ON event(kind,author);
+reindex;
+pragma optimize;
+PRAGMA user_version = 13;
+"##;
+    match conn.execute_batch(upgrade_sql) {
+        Ok(()) => {
+            info!("database schema upgraded v12 -> v13");
+        }
+        Err(err) => {
+            error!("update failed: {}", err);
+            panic!("database could not be upgraded");
+        }
+    }
+    Ok(13)
+}

src/server.rs

@@ -25,10 +25,12 @@ use hyper::{
 use rusqlite::OpenFlags;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
+use tokio::sync::Mutex;
 use std::collections::HashMap;
 use std::convert::Infallible;
 use std::net::SocketAddr;
 use std::path::Path;
+use std::sync::Arc;
 use std::sync::atomic::Ordering;
 use std::sync::mpsc::Receiver as MpscReceiver;
 use std::time::Duration;
@@ -54,6 +56,7 @@ async fn handle_web_request(
     broadcast: Sender<Event>,
     event_tx: tokio::sync::mpsc::Sender<SubmittedEvent>,
     shutdown: Receiver<()>,
+    safe_to_read: Arc<Mutex<u64>>,
 ) -> Result<Response<Body>, Infallible> {
     match (
         request.uri().path(),
@@ -76,6 +79,7 @@ async fn handle_web_request(
                             Ok(upgraded) => {
                                 // set WebSocket configuration options
                                 let config = WebSocketConfig {
+                                    max_send_queue: Some(1024),
                                     max_message_size: settings.limits.max_ws_message_bytes,
                                     max_frame_size: settings.limits.max_ws_frame_bytes,
                                     ..Default::default()
@@ -114,6 +118,7 @@ async fn handle_web_request(
                                     broadcast,
                                     event_tx,
                                     shutdown,
+				    safe_to_read,
                                 ));
                             }
                             // todo: trace, don't print...
@@ -328,8 +333,13 @@ pub fn start_server(settings: Settings, shutdown_rx: MpscReceiver<()>) -> Result
             2,
             false,
         );
-        db::db_optimize(maintenance_pool.clone()).await;
+
-        db::db_checkpoint(maintenance_pool).await;
+	// Create a mutex that will block readers, so that a
+	// checkpoint can be performed quickly.
+	let safe_to_read = Arc::new(Mutex::new(0));
+
+        db::db_optimize_task(maintenance_pool.clone()).await;
+        db::db_checkpoint_task(maintenance_pool, safe_to_read.clone()).await;
 
         // listen for (external to tokio) shutdown request
         let controlled_shutdown = invoke_shutdown.clone();
@@ -378,6 +388,7 @@ pub fn start_server(settings: Settings, shutdown_rx: MpscReceiver<()>) -> Result
             let event = event_tx.clone();
             let stop = invoke_shutdown.clone();
             let settings = settings.clone();
+            let safe_to_read = safe_to_read.clone();
             async move {
                 // service_fn converts our function into a `Service`
                 Ok::<_, Infallible>(service_fn(move |request: Request<Body>| {
@@ -389,6 +400,7 @@ pub fn start_server(settings: Settings, shutdown_rx: MpscReceiver<()>) -> Result
                         bcast.clone(),
                         event.clone(),
                         stop.subscribe(),
+			safe_to_read.clone(),
                     )
                 }))
             }
@@ -465,6 +477,7 @@ async fn nostr_server(
     broadcast: Sender<Event>,
     event_tx: mpsc::Sender<SubmittedEvent>,
     mut shutdown: Receiver<()>,
+    safe_to_read: Arc<Mutex<u64>>,
 ) {
     // the time this websocket nostr server started
     let orig_start = Instant::now();
@@ -674,8 +687,10 @@ async fn nostr_server(
                                     if let Some(previous_query) = running_queries.insert(s.id.to_owned(), abandon_query_tx) {
                     previous_query.send(()).ok();
                                     }
+		    if s.needs_historical_events() {
                                     // start a database query.  this spawns a blocking database query on a worker thread.
-                                    db::db_query(s, cid.to_owned(), pool.clone(), query_tx.clone(), abandon_query_rx).await;
+					db::db_query(s, cid.to_owned(), pool.clone(), query_tx.clone(), abandon_query_rx,safe_to_read.clone()).await;
+				    }
                 },
                 Err(e) => {
                     info!("Subscription error: {} (cid: {}, sub: {:?})", e, cid, s.id);

src/subscription.rs

@@ -200,6 +200,13 @@ impl Subscription {
     pub fn get_id(&self) -> String {
         self.id.clone()
     }
+
+    /// Determine if any filter is requesting historical (database)
+    /// queries.  If every filter has limit:0, we do not need to query the DB.
+    pub fn needs_historical_events(&self) -> bool {
+	self.filters.iter().any(|f| f.limit!=Some(0))
+    }
+
     /// Determine if this subscription matches a given [`Event`].  Any
     /// individual filter match is sufficient.
     pub fn interested_in_event(&self, event: &Event) -> bool {