fix(NIP-12): correctly search for mixed-case hex-like tags

Only lowercase and even-length tag values are stored as binary BLOBs.
Previously there was an error which search results from being returned
if the tag value was mixed-case and could be interpreted as hex.

A new database migration has been created to repair the `tag` table
for existing relays.

fixes: https://todo.sr.ht/~gheartsfield/nostr-rs-relay/37

src/schema.rs

@@ -1,13 +1,13 @@
 //! Database schema and migrations
 use crate::db::PooledConnection;
 use crate::error::Result;
-use crate::utils::is_hex;
+use crate::event::{single_char_tagname, Event};
+use crate::utils::is_lower_hex;
 use log::*;
 use rusqlite::limits::Limit;
 use rusqlite::params;
 use rusqlite::Connection;
-
-// TODO: drop the pubkey_ref and event_ref tables
+use std::time::Instant;
 
 /// Startup DB Pragmas
 pub const STARTUP_SQL: &str = r##"
@@ -24,7 +24,7 @@ PRAGMA journal_mode=WAL;
 PRAGMA main.synchronous=NORMAL;
 PRAGMA foreign_keys = ON;
 PRAGMA application_id = 1654008667;
-PRAGMA user_version = 5;
+PRAGMA user_version = 6;
 
 -- Event Table
 CREATE TABLE IF NOT EXISTS event (
@@ -53,7 +53,7 @@ id INTEGER PRIMARY KEY,
 event_id INTEGER NOT NULL, -- an event ID that contains a tag.
 name TEXT, -- the tag name ("p", "e", whatever)
 value TEXT, -- the tag value, if not hex.
-value_hex BLOB, -- the tag value, if it can be interpreted as a hex string.
+value_hex BLOB, -- the tag value, if it can be interpreted as a lowercase hex string.
 FOREIGN KEY(event_id) REFERENCES event(id) ON UPDATE CASCADE ON DELETE CASCADE
 );
 CREATE INDEX IF NOT EXISTS tag_val_index ON tag(value);
@@ -103,7 +103,7 @@ pub fn upgrade_db(conn: &mut PooledConnection) -> Result<()> {
     if curr_version == 0 {
         match conn.execute_batch(INIT_SQL) {
             Ok(()) => {
-                info!("database pragma/schema initialized to v4, and ready");
+                info!("database pragma/schema initialized to v6, and ready");
             }
             Err(err) => {
                 error!("update failed: {}", err);
@@ -154,7 +154,6 @@ PRAGMA user_version = 3;
                 panic!("database could not be upgraded");
             }
         }
-        info!("Starting transaction");
         // iterate over every event/pubkey tag
         let tx = conn.transaction()?;
         {
@@ -166,7 +165,7 @@ PRAGMA user_version = 3;
                 let tag_name: String = row.get(1)?;
                 let tag_value: String = row.get(2)?;
                 // this will leave behind p/e tags that were non-hex, but they are invalid anyways.
-                if is_hex(&tag_value) {
+                if is_lower_hex(&tag_value) {
                     tx.execute(
                         "INSERT INTO tag (event_id, name, value_hex) VALUES (?1, ?2, ?3);",
                         params![event_id, tag_name, hex::decode(&tag_value).ok()],
@@ -225,9 +224,63 @@ PRAGMA user_version=5;
             }
         }
     } else if curr_version == 5 {
-        debug!("Database version was already current");
-    } else if curr_version > 5 {
-        panic!("Database version is newer than supported by this executable");
+        info!("database schema needs update from 5->6");
+        // We need to rebuild the tags table.  iterate through the
+        // event table.  build event from json, insert tags into a
+        // fresh tag table.  This was needed due to a logic error in
+        // how hex-like tags got indexed.
+        let start = Instant::now();
+        let tx = conn.transaction()?;
+        {
+            // Clear out table
+            tx.execute("DELETE FROM tag;", [])?;
+            let mut stmt = tx.prepare("select id, content from event order by id;")?;
+            let mut tag_rows = stmt.query([])?;
+            while let Some(row) = tag_rows.next()? {
+                // we want to capture the event_id that had the tag, the tag name, and the tag hex value.
+                let event_id: u64 = row.get(0)?;
+                let event_json: String = row.get(1)?;
+                let event: Event = serde_json::from_str(&event_json)?;
+                // look at each event, and each tag, creating new tag entries if appropriate.
+                for t in event.tags.iter().filter(|x| x.len() > 1) {
+                    let tagname = t.get(0).unwrap();
+                    let tagnamechar_opt = single_char_tagname(tagname);
+                    if tagnamechar_opt.is_none() {
+                        continue;
+                    }
+                    // safe because len was > 1
+                    let tagval = t.get(1).unwrap();
+                    // insert as BLOB if we can restore it losslessly.
+                    // this means it needs to be even length and lowercase.
+                    if (tagval.len() % 2 == 0) && is_lower_hex(&tagval) {
+                        tx.execute(
+                            "INSERT INTO tag (event_id, name, value_hex) VALUES (?1, ?2, ?3);",
+                            params![event_id, tagname, hex::decode(&tagval).ok()],
+                        )?;
+                    } else {
+                        // otherwise, insert as text
+                        tx.execute(
+                            "INSERT INTO tag (event_id, name, value) VALUES (?1, ?2, ?3);",
+                            params![event_id, tagname, &tagval],
+                        )?;
+                    }
+                }
+            }
+            tx.execute("PRAGMA user_version = 6;", [])?;
+        }
+        tx.commit()?;
+        info!("database schema upgraded v5 -> v6 in {:?}", start.elapsed());
+        // vacuum after large table modification
+        let start = Instant::now();
+        conn.execute("VACUUM;", [])?;
+        info!("vacuumed DB after tags rebuild in {:?}", start.elapsed());
+    } else if curr_version == 6 {
+        debug!("Database version was already current (v6)");
+    } else if curr_version > 7 {
+        panic!(
+            "Database version is newer than supported by this executable (v{})",
+            curr_version
+        );
     }
 
     // Setup PRAGMA