From f682bd30297e067001deab04202cce8e75dbe291 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20Sch=C3=A4rtl?= <andreas@schaertl.me>
Date: Mon, 11 May 2020 14:00:40 +0200
Subject: [PATCH] ulocollect: crawl the file system
---
ulocollect/core/dummy_importer.go | 13 +++++++
ulocollect/core/fs_job.go | 57 +++++++++++++++++++++++++++++++
ulocollect/core/importer.go | 4 +--
ulocollect/main.go | 4 ++-
4 files changed, 74 insertions(+), 4 deletions(-)
create mode 100644 ulocollect/core/dummy_importer.go
create mode 100644 ulocollect/core/fs_job.go
diff --git a/ulocollect/core/dummy_importer.go b/ulocollect/core/dummy_importer.go
new file mode 100644
index 0000000..52d6383
--- /dev/null
+++ b/ulocollect/core/dummy_importer.go
@@ -0,0 +1,13 @@
+package core
+
+import (
+ "io"
+ "log"
+)
+
+type DummyImporter struct{}
+
+func (im DummyImporter) Import(rdf io.Reader) error {
+ log.Print("Import()")
+ return nil
+}
diff --git a/ulocollect/core/fs_job.go b/ulocollect/core/fs_job.go
new file mode 100644
index 0000000..8aa8d77
--- /dev/null
+++ b/ulocollect/core/fs_job.go
@@ -0,0 +1,57 @@
+package core
+
+import (
+ "bufio"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+)
+
+// Return a new Job that will scan path recursively for ULO/RDF files
+// and pass them to the executing Importer.
+func NewFileSystemJob(path string) *Job {
+ return &Job{
+ collectfunc: makeFsCollectFunc(path),
+ id: generateJobId(),
+ }
+}
+
+// Function that is called when collecting ULO/RDF from the local
+// file system.
+func makeFsCollectFunc(path string) func(*Job, *Collector) error {
+ return func(j *Job, c *Collector) error {
+ return filepath.Walk(path, func(path string, f os.FileInfo, err error) error {
+ if err != nil {
+ j.Logf("error walking path=%v: %v", path, err)
+ return nil
+ }
+
+ log.Println(path)
+ return importLocalFile(c.im, path)
+ })
+ }
+}
+
+// Import file at path with im. Returns no error if the file at
+// path is a not supported type.
+func importLocalFile(im Importer, path string) error {
+ if strings.HasSuffix(path, ".rdf") {
+ return importLocalRdfFile(im, path)
+ }
+
+ return nil
+}
+
+// Import ULO/RDF file at path with im.
+func importLocalRdfFile(im Importer, path string) error {
+ fd, err := os.Open(path)
+ if err != nil {
+ return err
+ }
+
+ defer fd.Close()
+
+ r := bufio.NewReader(fd)
+ return im.Import(r)
+}
diff --git a/ulocollect/core/importer.go b/ulocollect/core/importer.go
index ab13724..e1739df 100644
--- a/ulocollect/core/importer.go
+++ b/ulocollect/core/importer.go
@@ -1,8 +1,6 @@
package core
-import (
- "io"
-)
+import "io"
// Represents an Importer either locally or somewhere on the
// network.
diff --git a/ulocollect/main.go b/ulocollect/main.go
index 02cdaf0..61a6a59 100644
--- a/ulocollect/main.go
+++ b/ulocollect/main.go
@@ -10,7 +10,7 @@ import (
func main() {
log.SetFlags(log.LstdFlags | log.Lshortfile)
- j := core.NewDummyJob()
+ j := core.NewFileSystemJob("/etc")
c := core.Collector{}
if err := c.Submit(j); err != nil {
@@ -18,5 +18,7 @@ func main() {
}
fmt.Println("/done")
+
+ // TODO: make jobs awaitable
time.Sleep(5 * time.Second)
}
--
GitLab