diff options
author | Yorhel <git@yorhel.nl> | 2016-11-20 07:31:55 +0100 |
---|---|---|
committer | Yorhel <git@yorhel.nl> | 2016-11-20 07:31:55 +0100 |
commit | a1e5a2d80d4339b27c3c41ff97b3d31d9204d6b1 (patch) | |
tree | 2c5a8865a012c5f1c06662701c9d0261ce1e8510 /indexer | |
parent | 4bdd91f65e6565b568a19806c33ea870810c3be5 (diff) |
Indexer: Improve logging + cache management
Diffstat (limited to 'indexer')
-rw-r--r-- | indexer/src/main.rs | 5 | ||||
-rw-r--r-- | indexer/src/open.rs | 24 | ||||
-rw-r--r-- | indexer/src/pkg.rs | 14 |
3 files changed, 29 insertions, 14 deletions
diff --git a/indexer/src/main.rs b/indexer/src/main.rs index 84dd4bc..ef75984 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -72,6 +72,11 @@ fn main() { .filter(Some("postgres"), if verbose >= 4 { log::LogLevelFilter::Trace } else { log::LogLevelFilter::Info }) .init().unwrap(); + if let Err(e) = open::clear_cache() { + error!("Error clearing cache: {}", e); + return; + } + let dbhost = match std::env::var("MANNED_PG") { Ok(x) => x, Err(_) => { error!("MANNED_PG not set."); return } diff --git a/indexer/src/open.rs b/indexer/src/open.rs index 6bbe82d..39a9a3d 100644 --- a/indexer/src/open.rs +++ b/indexer/src/open.rs @@ -1,5 +1,5 @@ use std::io::{Read,Result,Error,ErrorKind,copy}; -use std::fs::{File,create_dir_all,metadata}; +use std::fs::{File,create_dir_all,metadata,read_dir,remove_file}; use std::hash::{Hash,Hasher,SipHasher}; use std::time::{Duration,SystemTime}; use url::Url; @@ -7,7 +7,7 @@ use hyper; const CACHE_PATH: &'static str = "/var/tmp/manned-indexer"; -const CACHE_TIME: u64 = 23*3600; +const CACHE_TIME: u64 = 20*3600; #[derive(Clone,Copy)] @@ -47,6 +47,19 @@ fn file(path: &str) -> Result<Box<Read>> { } +pub fn clear_cache() -> Result<()> { + create_dir_all(CACHE_PATH)?; + for f in read_dir(CACHE_PATH)? { + let f = f?.path(); + let m = metadata(&f)?; + if m.modified().unwrap() < SystemTime::now() - Duration::from_secs(CACHE_TIME) { + remove_file(&f)?; + } + } + Ok(()) +} + + impl<'a> Path<'a> { pub fn open(&self) -> Result<Box<Read>> { if let Ok(url) = Url::parse(self.path) { @@ -56,12 +69,9 @@ impl<'a> Path<'a> { if self.cache { let cfn = cache_fn(&url); - if let Ok(m) = metadata(&cfn) { - if m.modified().unwrap() > SystemTime::now() - Duration::from_secs(CACHE_TIME) { - return file(&cfn); - } + if let Ok(f) = file(&cfn) { + return Ok(f); } - try!(create_dir_all(CACHE_PATH)); { let mut rd = try!(fetch(url.as_str())); let mut wr = try!(File::create(&cfn)); diff --git a/indexer/src/pkg.rs b/indexer/src/pkg.rs index 11cfc9f..d480f2b 100644 --- a/indexer/src/pkg.rs +++ b/indexer/src/pkg.rs @@ -20,6 +20,8 @@ pub struct PkgOpt<'a> { fn insert_pkg(tr: &postgres::transaction::Transaction, opt: &PkgOpt) -> Option<i32> { + let pkginfo = format!("sys {} / {} / {} - {} @ {} @ {}", opt.sys, opt.cat, opt.pkg, opt.ver, opt.date, opt.file.path); + // The ON CONFLICT .. DO UPDATE is used instead of DO NOTHING because in that case the // RETURNING clause wouldn't give us a package id. let q = "INSERT INTO packages (system, category, name) VALUES($1, $2, $3) @@ -39,17 +41,17 @@ fn insert_pkg(tr: &postgres::transaction::Transaction, opt: &PkgOpt) -> Option<i if res.is_empty() { let q = "INSERT INTO package_versions (package, version, released, arch) VALUES($1, $2, $3::text::date, $4) RETURNING id"; verid = tr.query(q, &[&pkgid, &opt.ver, &opt.date, &opt.arch]).unwrap().get(0).get(0); - info!("New package pkgid {} verid {}", pkgid, verid); + info!("New package pkgid {} verid {}, {}", pkgid, verid, pkginfo); Some(verid) } else if opt.force { verid = res.get(0).get(0); - info!("Overwriting package pkgid {} verid {}", pkgid, verid); + info!("Overwriting package pkgid {} verid {}, {}", pkgid, verid, pkginfo); tr.query("DELETE FROM man WHERE package = $1", &[&verid]).unwrap(); Some(verid) } else { - info!("Package already in database, pkgid {} verid {}", pkgid, res.get(0).get::<usize,i32>(0)); + debug!("Package already in database, pkgid {} verid {}, {}", pkgid, res.get(0).get::<usize,i32>(0), pkginfo); None } } @@ -86,7 +88,7 @@ fn insert_man(tr: &postgres::GenericConnection, verid: i32, paths: &[&str], ent: for path in paths { insert_man_row(tr, verid, path, enc, dig.as_ref()); - debug!("Inserted man page: {} ({})", path, enc); + info!("Inserted man page: {} ({})", path, enc); } } @@ -100,7 +102,7 @@ fn insert_link(tr: &postgres::GenericConnection, verid: i32, src: &str, dest: &s let hash: Vec<u8> = res.get(0).get(0); let enc: String = res.get(0).get(1); insert_man_row(tr, verid, src, &enc, &hash); - debug!("Inserted man link: {} -> {}", src, dest); + info!("Inserted man link: {} -> {}", src, dest); } @@ -149,8 +151,6 @@ fn index_pkg(tr: &postgres::GenericConnection, opt: &PkgOpt, verid: i32) -> std: pub fn pkg(conn: &postgres::GenericConnection, opt: PkgOpt) { - info!("Handling pkg: {} / {} / {} - {} @ {} @ {}", opt.sys, opt.cat, opt.pkg, opt.ver, opt.date, opt.file.path); - let tr = conn.transaction().unwrap(); tr.set_rollback(); |