Commit 9ed0c14c authored by Ulrich's avatar Ulrich

added tool for safe parallelism

parent 08b87faa
......@@ -38,7 +38,6 @@ pub fn main() {
let (tx, rx) = channel();
println!("Loading corpus from {}", document_path);
let mut mut_corpus = Corpus::new(document_path.clone());
let mut documents_vec = Vec::new();
......@@ -52,10 +51,10 @@ pub fn main() {
documents_vec.push(my_document.path.clone());
pool.execute(move || {
// pool.execute(move || {
read_document(my_document_path.clone(), &my_annotation_document, &my_config);
my_tx.send(my_document_path.clone()).unwrap();
});
// });
}
......@@ -65,8 +64,6 @@ pub fn main() {
let mut successful_documents = vec![];
println!("number of documents {}", number_of_documents);
let timeout = Duration::from_secs(200);
for i in 0..number_of_documents{
......@@ -80,8 +77,6 @@ pub fn main() {
}
}
println!("done");
let mut doc_panic = vec![];
for document in documents_vec{
......@@ -103,8 +98,6 @@ pub fn read_document(document_path : String, annotation_location : &str, config
return;
}
println!("Starting {}", document_path.clone());
let annotation_document = opt_doc.unwrap();
let corpus = Corpus::new(document_path.clone());
......
from concurrent.futures import ThreadPoolExecutor
import sys
import os
executor = ThreadPoolExecutor(max_workers=2)
if len(sys.argv) != 4:
print("usage: python3 start.py Documents Annotations Log")
exit()
documents = sys.argv[1]
annotations = sys.argv[2]
log = sys.argv[3]
if not os.path.isdir(documents):
print("argument " + documents + " is not a folder")
exit()
if not os.path.isdir(annotations):
print("argument " + annotations + " is not a folder")
exit()
def my_fun(f):
command = "cargo run --release --bin scoring " + documents + "/" + f + " " + annotations + "/" + f + " >>" + log
print(command)
os.system(command)
docs = os.listdir(documents)
executor.map(my_fun, docs)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment