Commit b172b1e8 authored by Ulrich's avatar Ulrich

parallel

parent 1da7dba6
......@@ -2,6 +2,8 @@ extern crate llamapun;
extern crate libxml;
extern crate senna;
extern crate spotter_lib;
extern crate threadpool;
use std::fs;
use std::io::Write;
......@@ -12,6 +14,9 @@ use spotter_lib::util::*;
use spotter_lib::data::*;
use libxml::tree::*;
use libxml::parser::Parser;
use std::time::Duration;
use threadpool::ThreadPool;
use std::sync::mpsc::channel;
//Creates Harvests for Tom Wiesing's BSc thesis
......@@ -28,68 +33,111 @@ pub fn main() {
let input_path = args[1].to_owned();
let output_path = args[2].to_owned();
search_dir(Path::new(&input_path), &input_path, &output_path, &config);
let paths = search_dir(Path::new(&input_path), &input_path, &output_path, &config);
let pool = ThreadPool::new(CORES);
let (tx, rx) = channel();
let number_of_documents = paths.len();
for path in paths{
let my_name = path.clone();
let my_original_path = input_path.clone();
let my_output_path = output_path.clone();
let my_config = config.clone();
let my_tx = tx.clone();
pool.execute(move || {
process_file(my_name.clone(), &my_original_path, &my_output_path, &my_config);
my_tx.send(my_name).unwrap();
});
}
let timeout = Duration::from_secs(20);
for i in 0..number_of_documents{
let res = rx.recv_timeout(timeout);
if res.is_ok(){
let succ_doc = res.unwrap();
println!("finished ({}/{}) {}", (i+1), number_of_documents, succ_doc);
}else{
println!("timeout");
}
}
}
pub fn search_dir(dir_path : &Path, original_path : &str, output_path : &str, config : &Config){
pub fn search_dir(dir_path : &Path, original_path : &str, output_path : &str, config : &Config) -> Vec<String>{
let opt_path = fs::read_dir(&dir_path);
if opt_path.is_err(){
println!("error reading dir {}", &dir_path.display());
return;
return Vec::new();
}
let dir_path = opt_path.unwrap();
let mut paths = Vec::new();
for path in dir_path {
let entry = path.unwrap();
let p = entry.path();
let name = p.display();
if entry.metadata().is_ok() && entry.metadata().unwrap().is_dir(){
search_dir(&p, original_path, output_path, config);
paths.append(&mut search_dir(&p, original_path, output_path, config));
}else{
let string = format!("{}",name);
if string.ends_with(".kat.xml") {
let relative_path = &string[original_path.len()..];
println!("Scoring {}", name);
let filename = string.split("/").last().unwrap();
let filename_without_suffix = filename.replace(".kat.xml","");
// println!("File {}", filename_without_suffix);
let harvest = read_document(&string, config, &filename_without_suffix);
let string = format!("{}", name);
if string.ends_with(".kat.xml"){
paths.push(string.clone());
}
let dir = &relative_path[..(relative_path.len()-filename.len())];
let mut output = format!("{}/{}", output_path, relative_path);
let output_dir = format!("{}/{}", output_path, dir);
}
}
output = output.replace(".kat.xml",".harvest");
return paths;
}
// println!("output localtion {}", output);
// println!("relativ path {}", relative_path);
// println!("dir {}", output_dir);
pub fn process_file(name : String, original_path : &str, output_path : &str, config : &Config) {
let string = format!("{}", name);
if string.ends_with(".kat.xml") {
let relative_path = &string[original_path.len()..];
let filename = string.split("/").last().unwrap();
let filename_without_suffix = filename.replace(".kat.xml", "");
// println!("File {}", filename_without_suffix);
let harvest = read_document(&string, config, &filename_without_suffix);
let r = std::fs::create_dir_all(&output_dir);
if r.is_err(){
panic!("Couldnt create dir {}", &output_dir);
}
let dir = &relative_path[..(relative_path.len() - filename.len())];
let mut output = format!("{}/{}", output_path, relative_path);
let path = Path::new(&output);
let display = path.display();
let mut file = match std::fs::File::create(&path) {
Err(_) => panic!("couldn't create {}",
display),
Ok(file) => file,
};
match file.write_all(harvest.as_bytes()) {
Err(_) => {
panic!("couldn't write to {}", display)
},
Ok(_) => print!(""),
}
}
let output_dir = format!("{}/{}", output_path, dir);
output = output.replace(".kat.xml", ".harvest");
// println!("output localtion {}", output);
// println!("relativ path {}", relative_path);
// println!("dir {}", output_dir);
let r = std::fs::create_dir_all(&output_dir);
if r.is_err() {
panic!("Couldnt create dir {}", &output_dir);
}
let path = Path::new(&output);
let display = path.display();
let mut file = match std::fs::File::create(&path) {
Err(_) => panic!("couldn't create {}",
display),
Ok(file) => file,
};
match file.write_all(harvest.as_bytes()) {
Err(_) => {
panic!("couldn't write to {}", display)
},
Ok(_) => print!(""),
}
}
}
......
......@@ -19,7 +19,7 @@ use libxml::tree::Document as DOM;
static KAT_QE : &'static str = "KAT_1_QuantityExpression";
pub static CORES : usize = 20;
pub static CORES : usize = 8;
/* Returns the content of the "annotation" child of a math node, which contains the tex-code */
pub fn get_tex_from_math(math_node : Node) -> String{
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment