summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorYorhel <git@yorhel.nl>2020-05-25 09:09:43 +0200
committerYorhel <git@yorhel.nl>2020-05-25 09:09:43 +0200
commit19cfc38597b51f9c04ce54954895bde65ce5eccd (patch)
tree646a2de6af832a3a5b357571a4fce844081515e9
parent38c1de17fae7ccda6ad44dd2530abbbd708303a8 (diff)
Use new Hash type from chifs-common
-rw-r--r--Cargo.lock8
-rw-r--r--Cargo.toml3
-rw-r--r--sql/schema.sql4
-rw-r--r--src/action_meta.rs32
-rw-r--r--src/dispatch.rs17
-rw-r--r--src/main.rs4
6 files changed, 38 insertions, 30 deletions
diff --git a/Cargo.lock b/Cargo.lock
index f513b32..acf2cb7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -158,8 +158,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "chifs-common"
version = "0.1.0"
-source = "git+https://code.blicky.net/chifs/chifs-common-rs#6a4a2ea6c5cb3ac90d53645842729604dd4453cb"
+source = "git+https://code.blicky.net/chifs/chifs-common-rs#e955c5da130cb3af25098186722a7ee306beab05"
dependencies = [
+ "arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bendy 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "blake3 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)",
"colored 1.9.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -169,6 +173,7 @@ dependencies = [
"httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"multisock 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "postgres-types 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"syslog 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"zstd 0.4.28+zstd.1.4.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
@@ -178,7 +183,6 @@ name = "chifs-hub"
version = "0.1.0"
dependencies = [
"bendy 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "blake3 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"chifs-common 0.1.0 (git+https://code.blicky.net/chifs/chifs-common-rs)",
"chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)",
diff --git a/Cargo.toml b/Cargo.toml
index 7e97fa7..0f37aec 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,9 +10,8 @@ edition = "2018"
[dependencies]
bendy = "0.3"
-blake3 = "0.3"
chrono = { version = "0.4" }
-chifs-common = { git = "https://code.blicky.net/chifs/chifs-common-rs" }
+chifs-common = { features = ["postgres"], git = "https://code.blicky.net/chifs/chifs-common-rs" }
clap = { version = "2.33", default-features = false }
data-encoding = "2.2"
fallible-iterator = "0.2"
diff --git a/sql/schema.sql b/sql/schema.sql
index 8d4f775..f1751fb 100644
--- a/sql/schema.sql
+++ b/sql/schema.sql
@@ -17,8 +17,8 @@ CREATE TABLE params (
index_new_share boolean not null default true,
-- Automatically generated strong password, hashed with blake3. (I know,
-- worst password hashing function ever, but the password itself is 96
- -- bits, brute-forcing that will take some time, even with blake2)
- admin_pass bytea
+ -- bits, brute-forcing that will take some time, even with blake3)
+ admin_pass hash256
);
INSERT INTO params (schemaver) VALUES (1);
diff --git a/src/action_meta.rs b/src/action_meta.rs
index 4e31e1b..9763ede 100644
--- a/src/action_meta.rs
+++ b/src/action_meta.rs
@@ -1,5 +1,8 @@
use std::io::Read;
use std::time::Instant;
+use std::convert::TryFrom;
+use bendy::decoding::FromBencode;
+use chifs_common::Hash;
use crate::httpclient::{Client,Error};
use crate::config::Config;
@@ -10,7 +13,7 @@ const MAXMETALEN : u64 = 1024*1024;
const MAXPACKSLEN : u64 = 64*1024*1024; // 64 MiB is 2 million pack files. Ought to be enough, right?
-fn fetch_meta(conn: &Client) -> Result<(Option<String>,Option<String>,Vec<u8>), Error> {
+fn fetch_meta(conn: &Client) -> Result<(Option<String>,Option<String>,Hash), Error> {
let t_start = Instant::now();
conn.get("/.chifs-share/meta")?;
@@ -37,7 +40,7 @@ fn fetch_meta(conn: &Client) -> Result<(Option<String>,Option<String>,Vec<u8>),
match k {
b"title" => title = Some(String::from_utf8(v.try_into_bytes()?.to_vec()).map_err(|_| Error::Custom("Invalid UTF-8 in title field".to_string()))?),
b"contact" => contact = Some(String::from_utf8(v.try_into_bytes()?.to_vec()).map_err(|_| Error::Custom("Invalid UTF-8 in contact field".to_string()))?),
- b"blake3.packs" => packs = Some(v.try_into_bytes()?.to_vec()),
+ b"blake3.packs" => packs = Some(Hash::decode_bencode_object(v)?),
_ => ()
}
}
@@ -47,7 +50,7 @@ fn fetch_meta(conn: &Client) -> Result<(Option<String>,Option<String>,Vec<u8>),
}
-fn fetch_packs(conn: &Client, blake3: &[u8], buf: &mut Vec<u8>) -> Result<(), Error> {
+fn fetch_packs(conn: &Client, blake3: &Hash, buf: &mut Vec<u8>) -> Result<(), Error> {
let t_start = Instant::now();
conn.get("/.chifs-share/blake3.packs")?;
let res = conn.recv(MAXPACKSLEN)?;
@@ -60,21 +63,21 @@ fn fetch_packs(conn: &Client, blake3: &[u8], buf: &mut Vec<u8>) -> Result<(), Er
if buf.len() % 32 != 0 {
return Err(Error::Custom("blake3.packs size is not a multiple of 32".to_string()));
}
- if blake3::hash(&buf[..]).as_bytes() != blake3 {
+ if &Hash::hash_buf(&buf[..]) != blake3 {
return Err(Error::Custom("blake3.packs does not match the blake3 hash indicated in the metadata".to_string()));
}
Ok(())
}
-fn update(conf: &Config, db: &Pool, shareid: i32, addr: &str, blake3: Option<&[u8]>) -> Result<(), Error> {
+fn update(conf: &Config, db: &Pool, shareid: i32, addr: &str, blake3: Option<Hash>) -> Result<(), Error> {
let conn = Client::socks_connect(&conf.tor_address, addr)?;
let (title, contact, new_blake3) = fetch_meta(&conn)?;
let mut packs : Vec<u8> = Vec::new();
- if blake3 != Some(&new_blake3[..]) {
- if db.get().query_one("SELECT 1 FROM pack_indices WHERE blake3 = $1::bytea::hash256 LIMIT 1", &[&new_blake3]).is_err() {
- fetch_packs(&conn, &new_blake3[..], &mut packs)?;
+ if blake3 != Some(new_blake3) {
+ if db.get().query_one("SELECT 1 FROM pack_indices WHERE blake3 = $1 LIMIT 1", &[&new_blake3]).is_err() {
+ fetch_packs(&conn, &new_blake3, &mut packs)?;
}
}
@@ -83,9 +86,10 @@ fn update(conf: &Config, db: &Pool, shareid: i32, addr: &str, blake3: Option<&[u
let t_ins = Instant::now();
if !packs.is_empty() {
- let q_ind = txn.prepare("INSERT INTO pack_indices (blake3, pack) VALUES ($1::bytea::hash256, $2::bytea::hash256) ON CONFLICT (blake3, pack) DO NOTHING").unwrap();
- let q_packs = txn.prepare("INSERT INTO packs (pack) VALUES ($1::bytea::hash256) ON CONFLICT (pack) DO NOTHING").unwrap();
+ let q_ind = txn.prepare("INSERT INTO pack_indices (blake3, pack) VALUES ($1, $2) ON CONFLICT (blake3, pack) DO NOTHING").unwrap();
+ let q_packs = txn.prepare("INSERT INTO packs (pack) VALUES ($1) ON CONFLICT (pack) DO NOTHING").unwrap();
for p in packs.chunks_exact(32) {
+ let p = Hash::try_from(p).unwrap();
txn.execute(&q_ind, &[&new_blake3, &p]).unwrap();
txn.execute(&q_packs, &[&p]).unwrap();
}
@@ -93,16 +97,16 @@ fn update(conf: &Config, db: &Pool, shareid: i32, addr: &str, blake3: Option<&[u
let t_upd = Instant::now();
txn.execute(
- "UPDATE shares SET title = $1, contact = $2, blake3_packs = $3::bytea::hash256 WHERE id = $4 AND (title,contact,blake3_packs) IS DISTINCT FROM ($1,$2,$3::bytea::hash256)",
+ "UPDATE shares SET title = $1, contact = $2, blake3_packs = $3 WHERE id = $4 AND (title,contact,blake3_packs) IS DISTINCT FROM ($1,$2,$3)",
&[&title, &contact, &new_blake3, &shareid]
).unwrap();
let t_del = Instant::now();
- if blake3 != Some(&new_blake3[..]) {
+ if blake3 != Some(new_blake3) {
// Delete unreferenced pack_indices, packs and files. The latter is done automatically as
// part of an ON DELETE foreign key reference.
txn.execute("
- WITH del(blake3) AS (SELECT $1::bytea::hash256 WHERE NOT EXISTS(SELECT 1 FROM shares WHERE blake3_packs = $1::bytea::hash256))
+ WITH del(blake3) AS (SELECT $1::hash256 WHERE NOT EXISTS(SELECT 1 FROM shares WHERE blake3_packs = $1))
, x AS (
DELETE FROM packs
WHERE pack IN(SELECT pack FROM pack_indices WHERE blake3 IN(SELECT blake3 FROM del))
@@ -125,7 +129,7 @@ fn update(conf: &Config, db: &Pool, shareid: i32, addr: &str, blake3: Option<&[u
}
-pub fn run(conf: &Config, db: Pool, shareid: i32, addr: &str, blake3: Option<&[u8]>) {
+pub fn run(conf: &Config, db: Pool, shareid: i32, addr: &str, blake3: Option<Hash>) {
debug!("Fetching Meta for share {} ({})", shareid, addr);
share_log::Action::Meta.run(db.clone(), shareid, |_| update(conf, &db, shareid, addr, blake3));
}
diff --git a/src/dispatch.rs b/src/dispatch.rs
index ceaa7c2..4378f2e 100644
--- a/src/dispatch.rs
+++ b/src/dispatch.rs
@@ -1,10 +1,11 @@
use std::fmt::Display;
-use std::hash::Hash;
+use std::hash::Hash as StdHash;
use std::sync::{Arc,Mutex};
use std::collections::HashSet;
use std::time::Duration;
//use chrono::prelude::*;
use fallible_iterator::FallibleIterator;
+use chifs_common::Hash;
use crate::db::Pool;
use crate::config::Config;
@@ -14,16 +15,16 @@ use crate::action_meta;
// Not sure why I generalized this, T is prolly always going to be an i32.
-struct TaskGuard<T: Display + Hash + Eq + Copy + Clone> {
+struct TaskGuard<T: Display + StdHash + Eq + Copy + Clone> {
lst: Arc<Mutex<HashSet<T>>>,
id: T
}
-impl<T: Display + Hash + Eq + Copy + Clone> TaskGuard<T> {
+impl<T: Display + StdHash + Eq + Copy + Clone> TaskGuard<T> {
fn done(self) {} // Implicitely calls drop
}
-impl<T: Display + Hash + Eq + Copy + Clone> Drop for TaskGuard<T> {
+impl<T: Display + StdHash + Eq + Copy + Clone> Drop for TaskGuard<T> {
fn drop(&mut self) {
if std::thread::panicking() {
error!("Thread for task {} has panicked", self.id);
@@ -35,7 +36,7 @@ impl<T: Display + Hash + Eq + Copy + Clone> Drop for TaskGuard<T> {
fn handle_queue<S,T,F,G>(action: &str, num: i32, conn: &mut postgres::Client, queue: &postgres::Statement, tasks: &Arc<Mutex<HashSet<T>>>, f: F, cmd: G) -> Result<(),postgres::Error>
- where T: Display + Hash + Eq + Copy + Clone + Send + 'static,
+ where T: Display + StdHash + Eq + Copy + Clone + Send + 'static,
F: Fn(&postgres::row::Row) -> (T, S),
G: FnOnce(S) + Send + Copy + 'static,
S: Send + 'static
@@ -93,7 +94,7 @@ pub fn run(conf: Arc<Config>, db: Pool) {
// action is lagging behind. It also prevents losing updates to a race condition: If we fetch
// new data while old data is being indexed, it's possible that 'next_index' is being reset to
// NULL even when we do have new data to index. Yay, parallelism.
- let meta_q = conn.prepare("SELECT id, addr, blake3_packs::bytea FROM shares WHERE next_meta <= NOW() ORDER BY next_meta LIMIT $1::int").unwrap();
+ let meta_q = conn.prepare("SELECT id, addr, blake3_packs FROM shares WHERE next_meta <= NOW() ORDER BY next_meta LIMIT $1::int").unwrap();
//let fetch_q = conn.prepare("SELECT id, addr FROM shares WHERE next_fetch <= NOW() AND next_index IS NULL ORDER BY next_fetch LIMIT $1::int").unwrap();
//let index_q = conn.prepare("SELECT id, addr, next_index FROM shares WHERE next_index <= NOW() ORDER BY next_index LIMIT $1::int").unwrap();
@@ -104,8 +105,8 @@ pub fn run(conf: Arc<Config>, db: Pool) {
handle_queue("Meta",
conf.meta_processes, &mut conn, &meta_q, &meta_tasks,
|row| (row.get(0), (conf.clone(), db.clone(), row.get(0), row.get(1), row.get(2))),
- move |data: (Arc<Config>, Pool, i32, String, Option<Vec<u8>>)| {
- action_meta::run(&data.0, data.1, data.2, &data.3[..], data.4.as_deref());
+ move |data: (Arc<Config>, Pool, i32, String, Option<Hash>)| {
+ action_meta::run(&data.0, data.1, data.2, &data.3[..], data.4);
}
).unwrap();
diff --git a/src/main.rs b/src/main.rs
index 8a8c9b5..2154746 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -99,8 +99,8 @@ fn main() {
getrandom::getrandom(&mut pass).expect("Unable to obtain OS randomness");
let pass = data_encoding::BASE64URL_NOPAD.encode(&pass[..]);
- let b3 = blake3::hash(pass.as_bytes());
- db.get().execute("UPDATE params SET admin_pass = $1", &[&&b3.as_bytes()[..]]).unwrap();
+ let b3 = chifs_common::Hash::hash_buf(pass.as_bytes());
+ db.get().execute("UPDATE params SET admin_pass = $1", &[&b3]).unwrap();
println!("The admin password has been set to: {}", pass);
} else {