summaryrefslogtreecommitdiff
path: root/indexer
diff options
context:
space:
mode:
authorYorhel <git@yorhel.nl>2016-11-20 07:31:55 +0100
committerYorhel <git@yorhel.nl>2016-11-20 07:31:55 +0100
commita1e5a2d80d4339b27c3c41ff97b3d31d9204d6b1 (patch)
tree2c5a8865a012c5f1c06662701c9d0261ce1e8510 /indexer
parent4bdd91f65e6565b568a19806c33ea870810c3be5 (diff)
Indexer: Improve logging + cache management
Diffstat (limited to 'indexer')
-rw-r--r--indexer/src/main.rs5
-rw-r--r--indexer/src/open.rs24
-rw-r--r--indexer/src/pkg.rs14
3 files changed, 29 insertions, 14 deletions
diff --git a/indexer/src/main.rs b/indexer/src/main.rs
index 84dd4bc..ef75984 100644
--- a/indexer/src/main.rs
+++ b/indexer/src/main.rs
@@ -72,6 +72,11 @@ fn main() {
.filter(Some("postgres"), if verbose >= 4 { log::LogLevelFilter::Trace } else { log::LogLevelFilter::Info })
.init().unwrap();
+ if let Err(e) = open::clear_cache() {
+ error!("Error clearing cache: {}", e);
+ return;
+ }
+
let dbhost = match std::env::var("MANNED_PG") {
Ok(x) => x,
Err(_) => { error!("MANNED_PG not set."); return }
diff --git a/indexer/src/open.rs b/indexer/src/open.rs
index 6bbe82d..39a9a3d 100644
--- a/indexer/src/open.rs
+++ b/indexer/src/open.rs
@@ -1,5 +1,5 @@
use std::io::{Read,Result,Error,ErrorKind,copy};
-use std::fs::{File,create_dir_all,metadata};
+use std::fs::{File,create_dir_all,metadata,read_dir,remove_file};
use std::hash::{Hash,Hasher,SipHasher};
use std::time::{Duration,SystemTime};
use url::Url;
@@ -7,7 +7,7 @@ use hyper;
const CACHE_PATH: &'static str = "/var/tmp/manned-indexer";
-const CACHE_TIME: u64 = 23*3600;
+const CACHE_TIME: u64 = 20*3600;
#[derive(Clone,Copy)]
@@ -47,6 +47,19 @@ fn file(path: &str) -> Result<Box<Read>> {
}
+pub fn clear_cache() -> Result<()> {
+ create_dir_all(CACHE_PATH)?;
+ for f in read_dir(CACHE_PATH)? {
+ let f = f?.path();
+ let m = metadata(&f)?;
+ if m.modified().unwrap() < SystemTime::now() - Duration::from_secs(CACHE_TIME) {
+ remove_file(&f)?;
+ }
+ }
+ Ok(())
+}
+
+
impl<'a> Path<'a> {
pub fn open(&self) -> Result<Box<Read>> {
if let Ok(url) = Url::parse(self.path) {
@@ -56,12 +69,9 @@ impl<'a> Path<'a> {
if self.cache {
let cfn = cache_fn(&url);
- if let Ok(m) = metadata(&cfn) {
- if m.modified().unwrap() > SystemTime::now() - Duration::from_secs(CACHE_TIME) {
- return file(&cfn);
- }
+ if let Ok(f) = file(&cfn) {
+ return Ok(f);
}
- try!(create_dir_all(CACHE_PATH));
{
let mut rd = try!(fetch(url.as_str()));
let mut wr = try!(File::create(&cfn));
diff --git a/indexer/src/pkg.rs b/indexer/src/pkg.rs
index 11cfc9f..d480f2b 100644
--- a/indexer/src/pkg.rs
+++ b/indexer/src/pkg.rs
@@ -20,6 +20,8 @@ pub struct PkgOpt<'a> {
fn insert_pkg(tr: &postgres::transaction::Transaction, opt: &PkgOpt) -> Option<i32> {
+ let pkginfo = format!("sys {} / {} / {} - {} @ {} @ {}", opt.sys, opt.cat, opt.pkg, opt.ver, opt.date, opt.file.path);
+
// The ON CONFLICT .. DO UPDATE is used instead of DO NOTHING because in that case the
// RETURNING clause wouldn't give us a package id.
let q = "INSERT INTO packages (system, category, name) VALUES($1, $2, $3)
@@ -39,17 +41,17 @@ fn insert_pkg(tr: &postgres::transaction::Transaction, opt: &PkgOpt) -> Option<i
if res.is_empty() {
let q = "INSERT INTO package_versions (package, version, released, arch) VALUES($1, $2, $3::text::date, $4) RETURNING id";
verid = tr.query(q, &[&pkgid, &opt.ver, &opt.date, &opt.arch]).unwrap().get(0).get(0);
- info!("New package pkgid {} verid {}", pkgid, verid);
+ info!("New package pkgid {} verid {}, {}", pkgid, verid, pkginfo);
Some(verid)
} else if opt.force {
verid = res.get(0).get(0);
- info!("Overwriting package pkgid {} verid {}", pkgid, verid);
+ info!("Overwriting package pkgid {} verid {}, {}", pkgid, verid, pkginfo);
tr.query("DELETE FROM man WHERE package = $1", &[&verid]).unwrap();
Some(verid)
} else {
- info!("Package already in database, pkgid {} verid {}", pkgid, res.get(0).get::<usize,i32>(0));
+ debug!("Package already in database, pkgid {} verid {}, {}", pkgid, res.get(0).get::<usize,i32>(0), pkginfo);
None
}
}
@@ -86,7 +88,7 @@ fn insert_man(tr: &postgres::GenericConnection, verid: i32, paths: &[&str], ent:
for path in paths {
insert_man_row(tr, verid, path, enc, dig.as_ref());
- debug!("Inserted man page: {} ({})", path, enc);
+ info!("Inserted man page: {} ({})", path, enc);
}
}
@@ -100,7 +102,7 @@ fn insert_link(tr: &postgres::GenericConnection, verid: i32, src: &str, dest: &s
let hash: Vec<u8> = res.get(0).get(0);
let enc: String = res.get(0).get(1);
insert_man_row(tr, verid, src, &enc, &hash);
- debug!("Inserted man link: {} -> {}", src, dest);
+ info!("Inserted man link: {} -> {}", src, dest);
}
@@ -149,8 +151,6 @@ fn index_pkg(tr: &postgres::GenericConnection, opt: &PkgOpt, verid: i32) -> std:
pub fn pkg(conn: &postgres::GenericConnection, opt: PkgOpt) {
- info!("Handling pkg: {} / {} / {} - {} @ {} @ {}", opt.sys, opt.cat, opt.pkg, opt.ver, opt.date, opt.file.path);
-
let tr = conn.transaction().unwrap();
tr.set_rollback();