summaryrefslogtreecommitdiff
path: root/indexer
diff options
context:
space:
mode:
authorYorhel <git@yorhel.nl>2016-12-18 15:05:25 +0100
committerYorhel <git@yorhel.nl>2016-12-18 15:08:56 +0100
commitd1530045326e0c82cea93d2997932a3b4a019dd8 (patch)
tree4ae5ee5261f096a5ff789bb7e3187e8f4a2b3457 /indexer
parentb9764fce4a57bbbaff765fdd36339b97dc2726c2 (diff)
indexer: Support FreeBSD 9.3+; remove now obsolete add_index.pl
Diffstat (limited to 'indexer')
-rw-r--r--indexer/src/main.rs14
-rw-r--r--indexer/src/pkg.rs6
-rw-r--r--indexer/src/sys_freebsd2.rs63
3 files changed, 82 insertions, 1 deletions
diff --git a/indexer/src/main.rs b/indexer/src/main.rs
index fe65637..ee6976a 100644
--- a/indexer/src/main.rs
+++ b/indexer/src/main.rs
@@ -20,6 +20,7 @@ mod pkg;
mod sys_arch;
mod sys_deb;
mod sys_freebsd1;
+mod sys_freebsd2;
// Convenience function to get a system id by short-name. Panics if the system doesn't exist.
@@ -67,6 +68,11 @@ fn main() {
(@arg mirror: --mirror +required +takes_value "Mirror URL (should point to the packages/ dir)")
(@arg arch: --arch +required +takes_value "Arch")
)
+ (@subcommand freebsd2 =>
+ (about: "Index packages from a FreeBSD >= 9.3 package repo")
+ (@arg sys: --sys +required +takes_value "System short-name")
+ (@arg mirror: --mirror +required +takes_value "Mirror URL")
+ )
).get_matches();
unsafe { pkg::DRY_RUN = arg.is_present("dry") };
@@ -101,6 +107,7 @@ fn main() {
let date = match matches.value_of("date").unwrap() {
"deb" => pkg::Date::Deb,
"desc" => pkg::Date::Desc,
+ "max" => pkg::Date::Max,
s => pkg::Date::Known(s),
};
pkg::pkg(&db, pkg::PkgOpt {
@@ -140,5 +147,12 @@ fn main() {
).unwrap_or_else(|e| error!("{}", e));
}
+ if let Some(matches) = arg.subcommand_matches("freebsd2") {
+ sys_freebsd2::sync(&db,
+ sysbyshort(&db, matches.value_of("sys").unwrap()),
+ matches.value_of("mirror").unwrap()
+ ).unwrap_or_else(|e| error!("{}", e));
+ }
+
trace!("Exiting");
}
diff --git a/indexer/src/pkg.rs b/indexer/src/pkg.rs
index 3caabfd..907b670 100644
--- a/indexer/src/pkg.rs
+++ b/indexer/src/pkg.rs
@@ -17,6 +17,8 @@ pub enum Date<'a> {
Found(i64), // Found in package
Deb, // Should be read from the timestamp of the 'debian-binary' file
Desc, // Should be read from the '+DESC' file (FreeBSD <= 9.2)
+ Max, // Use the latest timestamp in the archive
+ MaxVal(i64),
}
@@ -26,6 +28,8 @@ impl<'a> Date<'a> {
*self = match *self {
Date::Deb if ent.format() == Format::Ar && ent.path() == Some("debian-binary") => Date::Found(ent.mtime()),
Date::Desc if ent.path() == Some("+DESC") => Date::Found(ent.mtime()),
+ Date::Max => Date::MaxVal(ent.mtime()),
+ Date::MaxVal(t) if ent.mtime() > t => Date::MaxVal(ent.mtime()),
x => x,
}
}
@@ -193,7 +197,7 @@ fn index_pkg(tr: &postgres::GenericConnection, mut opt: PkgOpt, verid: i32) -> s
match opt.date {
Date::Known(_) => Ok(()),
- Date::Found(t) => {
+ Date::Found(t) | Date::MaxVal(t) => {
let date = NaiveDateTime::from_timestamp(t, 0).format("%Y-%m-%d").to_string();
debug!("Date from package: {}", date);
tr.execute("UPDATE package_versions SET released = $1::text::date WHERE id = $2", &[&date, &verid]).unwrap();
diff --git a/indexer/src/sys_freebsd2.rs b/indexer/src/sys_freebsd2.rs
new file mode 100644
index 0000000..cd49aab
--- /dev/null
+++ b/indexer/src/sys_freebsd2.rs
@@ -0,0 +1,63 @@
+use std::io::{BufReader,BufRead,Result,Error,ErrorKind};
+use regex::bytes::Regex;
+use std::str;
+use postgres;
+
+use open;
+use pkg;
+use archive::{Archive,ArchiveEntry};
+
+
+fn getpkgsite(mut ent: Option<ArchiveEntry>) -> Result<ArchiveEntry> {
+ while let Some(e) = ent {
+ if e.path() == Some("packagesite.yaml") {
+ return Ok(e)
+ }
+ ent = e.next()?
+ }
+ Err(Error::new(ErrorKind::Other, "No packagesite.yaml found"))
+}
+
+
+pub fn sync(pg: &postgres::GenericConnection, sys: i32, mirror: &str) -> Result<()> {
+ let path = format!("{}packagesite.txz", mirror);
+ let mut rd = open::Path{path: &path, cache: true, canbelocal: false}.open()?;
+
+ let ent = Archive::open_archive(&mut rd)?;
+ let brd = BufReader::new(getpkgsite(ent)?);
+
+ // It's technically a JSON/YAML file, but rather than bothering with a proper JSON parser,
+ // these regexes will do fine.
+ lazy_static!(
+ static ref RE_NAME : Regex = Regex::new(r#""name"\s*:\s*"(?u:([^ "]+))""#).unwrap();
+ static ref RE_VER : Regex = Regex::new(r#""version"\s*:\s*"(?u:([^ "]+))""#).unwrap();
+ static ref RE_CAT : Regex = Regex::new(r#""origin"\s*:\s*"(?u:([^ "/]+))"#).unwrap();
+ static ref RE_PATH : Regex = Regex::new(r#""path"\s*:\s*"(?u:([^ "]+))""#).unwrap();
+ static ref RE_ARCH : Regex = Regex::new(r#""arch"\s*:\s*"(?u:([^ "]+))""#).unwrap();
+ );
+
+ for line in brd.split(b'\n') {
+ let line = line?;
+ let name = match RE_NAME.captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() };
+ let ver = match RE_VER .captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() };
+ let cat = match RE_CAT .captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() };
+ let path = match RE_PATH.captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() };
+ let arch = match RE_ARCH.captures(&line) { None => continue, Some(c) => str::from_utf8(c.at(1).unwrap()).unwrap() };
+ let uri = format!("{}{}", mirror, path);
+ pkg::pkg(pg, pkg::PkgOpt{
+ force: false,
+ sys: sys,
+ cat: cat,
+ pkg: name,
+ ver: ver,
+ date: pkg::Date::Max,
+ arch: Some(arch),
+ file: open::Path{
+ path: &uri,
+ cache: false,
+ canbelocal: false,
+ },
+ });
+ }
+ Ok(())
+}