diff options
author | Yorhel <git@yorhel.nl> | 2016-12-18 15:05:25 +0100 |
---|---|---|
committer | Yorhel <git@yorhel.nl> | 2016-12-18 15:08:56 +0100 |
commit | d1530045326e0c82cea93d2997932a3b4a019dd8 (patch) | |
tree | 4ae5ee5261f096a5ff789bb7e3187e8f4a2b3457 /indexer | |
parent | b9764fce4a57bbbaff765fdd36339b97dc2726c2 (diff) |
indexer: Support FreeBSD 9.3+; remove now obsolete add_index.pl
Diffstat (limited to 'indexer')
-rw-r--r-- | indexer/src/main.rs | 14 | ||||
-rw-r--r-- | indexer/src/pkg.rs | 6 | ||||
-rw-r--r-- | indexer/src/sys_freebsd2.rs | 63 |
3 files changed, 82 insertions, 1 deletions
diff --git a/indexer/src/main.rs b/indexer/src/main.rs index fe65637..ee6976a 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -20,6 +20,7 @@ mod pkg; mod sys_arch; mod sys_deb; mod sys_freebsd1; +mod sys_freebsd2; // Convenience function to get a system id by short-name. Panics if the system doesn't exist. @@ -67,6 +68,11 @@ fn main() { (@arg mirror: --mirror +required +takes_value "Mirror URL (should point to the packages/ dir)") (@arg arch: --arch +required +takes_value "Arch") ) + (@subcommand freebsd2 => + (about: "Index packages from a FreeBSD >= 9.3 package repo") + (@arg sys: --sys +required +takes_value "System short-name") + (@arg mirror: --mirror +required +takes_value "Mirror URL") + ) ).get_matches(); unsafe { pkg::DRY_RUN = arg.is_present("dry") }; @@ -101,6 +107,7 @@ fn main() { let date = match matches.value_of("date").unwrap() { "deb" => pkg::Date::Deb, "desc" => pkg::Date::Desc, + "max" => pkg::Date::Max, s => pkg::Date::Known(s), }; pkg::pkg(&db, pkg::PkgOpt { @@ -140,5 +147,12 @@ fn main() { ).unwrap_or_else(|e| error!("{}", e)); } + if let Some(matches) = arg.subcommand_matches("freebsd2") { + sys_freebsd2::sync(&db, + sysbyshort(&db, matches.value_of("sys").unwrap()), + matches.value_of("mirror").unwrap() + ).unwrap_or_else(|e| error!("{}", e)); + } + trace!("Exiting"); } diff --git a/indexer/src/pkg.rs b/indexer/src/pkg.rs index 3caabfd..907b670 100644 --- a/indexer/src/pkg.rs +++ b/indexer/src/pkg.rs @@ -17,6 +17,8 @@ pub enum Date<'a> { Found(i64), // Found in package Deb, // Should be read from the timestamp of the 'debian-binary' file Desc, // Should be read from the '+DESC' file (FreeBSD <= 9.2) + Max, // Use the latest timestamp in the archive + MaxVal(i64), } @@ -26,6 +28,8 @@ impl<'a> Date<'a> { *self = match *self { Date::Deb if ent.format() == Format::Ar && ent.path() == Some("debian-binary") => Date::Found(ent.mtime()), Date::Desc if ent.path() == Some("+DESC") => Date::Found(ent.mtime()), + Date::Max => Date::MaxVal(ent.mtime()), + Date::MaxVal(t) if ent.mtime() > t => Date::MaxVal(ent.mtime()), x => x, } } @@ -193,7 +197,7 @@ fn index_pkg(tr: &postgres::GenericConnection, mut opt: PkgOpt, verid: i32) -> s match opt.date { Date::Known(_) => Ok(()), - Date::Found(t) => { + Date::Found(t) | Date::MaxVal(t) => { let date = NaiveDateTime::from_timestamp(t, 0).format("%Y-%m-%d").to_string(); debug!("Date from package: {}", date); tr.execute("UPDATE package_versions SET released = $1::text::date WHERE id = $2", &[&date, &verid]).unwrap(); diff --git a/indexer/src/sys_freebsd2.rs b/indexer/src/sys_freebsd2.rs new file mode 100644 index 0000000..cd49aab --- /dev/null +++ b/indexer/src/sys_freebsd2.rs @@ -0,0 +1,63 @@ +use std::io::{BufReader,BufRead,Result,Error,ErrorKind}; +use regex::bytes::Regex; +use std::str; +use postgres; + +use open; +use pkg; +use archive::{Archive,ArchiveEntry}; + + +fn getpkgsite(mut ent: Option<ArchiveEntry>) -> Result<ArchiveEntry> { + while let Some(e) = ent { + if e.path() == Some("packagesite.yaml") { + return Ok(e) + } + ent = e.next()? + } + Err(Error::new(ErrorKind::Other, "No packagesite.yaml found")) +} + + +pub fn sync(pg: &postgres::GenericConnection, sys: i32, mirror: &str) -> Result<()> { + let path = format!("{}packagesite.txz", mirror); + let mut rd = open::Path{path: &path, cache: true, canbelocal: false}.open()?; + + let ent = Archive::open_archive(&mut rd)?; + let brd = BufReader::new(getpkgsite(ent)?); + + // It's technically a JSON/YAML file, but rather than bothering with a proper JSON parser, + // these regexes will do fine. + lazy_static!( + static ref RE_NAME : Regex = Regex::new(r#""name"\s*:\s*"(?u:([^ "]+))""#).unwrap(); + static ref RE_VER : Regex = Regex::new(r#""version"\s*:\s*"(?u:([^ "]+))""#).unwrap(); + static ref RE_CAT : Regex = Regex::new(r#""origin"\s*:\s*"(?u:([^ "/]+))"#).unwrap(); + static ref RE_PATH : Regex = Regex::new(r#""path"\s*:\s*"(?u:([^ "]+))""#).unwrap(); + static ref RE_ARCH : Regex = Regex::new(r#""arch"\s*:\s*"(?u:([^ "]+))""#).unwrap(); + ); + + for line in brd.split(b'\n') { + let line = line?; + let name = match RE_NAME.captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() }; + let ver = match RE_VER .captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() }; + let cat = match RE_CAT .captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() }; + let path = match RE_PATH.captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() }; + let arch = match RE_ARCH.captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() }; + let uri = format!("{}{}", mirror, path); + pkg::pkg(pg, pkg::PkgOpt{ + force: false, + sys: sys, + cat: cat, + pkg: name, + ver: ver, + date: pkg::Date::Max, + arch: Some(arch), + file: open::Path{ + path: &uri, + cache: false, + canbelocal: false, + }, + }); + } + Ok(()) +} |