ulocollect: crawl the file system

ulocollect/core/dummy_importer.go

+package core
+
+import (
+	"io"
+	"log"
+)
+
+type DummyImporter struct{}
+
+func (im DummyImporter) Import(rdf io.Reader) error {
+	log.Print("Import()")
+	return nil
+}

ulocollect/core/fs_job.go

+package core
+
+import (
+	"bufio"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// Return a new Job that will scan path recursively for ULO/RDF files
+// and pass them to the executing Importer.
+func NewFileSystemJob(path string) *Job {
+	return &Job{
+		collectfunc: makeFsCollectFunc(path),
+		id:          generateJobId(),
+	}
+}
+
+// Function that is called when collecting ULO/RDF from the local
+// file system.
+func makeFsCollectFunc(path string) func(*Job, *Collector) error {
+	return func(j *Job, c *Collector) error {
+		return filepath.Walk(path, func(path string, f os.FileInfo, err error) error {
+			if err != nil {
+				j.Logf("error walking path=%v: %v", path, err)
+				return nil
+			}
+
+			log.Println(path)
+			return importLocalFile(c.im, path)
+		})
+	}
+}
+
+// Import file at path with im. Returns no error if the file at
+// path is a not supported type.
+func importLocalFile(im Importer, path string) error {
+	if strings.HasSuffix(path, ".rdf") {
+		return importLocalRdfFile(im, path)
+	}
+
+	return nil
+}
+
+// Import ULO/RDF file at path with im.
+func importLocalRdfFile(im Importer, path string) error {
+	fd, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+
+	defer fd.Close()
+
+	r := bufio.NewReader(fd)
+	return im.Import(r)
+}

ulocollect/core/importer.go

 package core
 
-import (
-	"io"
-)
+import "io"
 
 // Represents an Importer either locally or somewhere on the
 // network.

ulocollect/main.go

@@ -10,7 +10,7 @@ import (
 func main() {
 	log.SetFlags(log.LstdFlags | log.Lshortfile)
 
-	j := core.NewDummyJob()
+	j := core.NewFileSystemJob("/etc")
 	c := core.Collector{}
 
 	if err := c.Submit(j); err != nil {
@@ -18,5 +18,7 @@ func main() {
 	}
 
 	fmt.Println("/done")
+
+	// TODO: make jobs awaitable
 	time.Sleep(5 * time.Second)
 }