|
1
|
|
|
package operations |
|
2
|
|
|
|
|
3
|
|
|
import ( |
|
4
|
|
|
"encoding/csv" |
|
5
|
|
|
"fmt" |
|
6
|
|
|
"github.com/vvval/go-metadata-scanner/config" |
|
7
|
|
|
"github.com/vvval/go-metadata-scanner/util/log" |
|
8
|
|
|
"github.com/vvval/go-metadata-scanner/vars" |
|
9
|
|
|
"github.com/vvval/go-metadata-scanner/vars/metadata" |
|
10
|
|
|
"io" |
|
11
|
|
|
) |
|
12
|
|
|
|
|
13
|
|
|
func ReadCSV(reader *csv.Reader, dict config.DictConfig, callback func(filename string, payload metadata.Payload)) { |
|
14
|
|
|
var columnsFound bool |
|
15
|
|
|
var columns map[int]vars.Tag |
|
16
|
|
|
|
|
17
|
|
|
var i int |
|
18
|
|
|
for { |
|
19
|
|
|
i++ |
|
20
|
|
|
|
|
21
|
|
|
row, err := readIntoMap(reader) |
|
22
|
|
|
if err == io.EOF { |
|
23
|
|
|
break |
|
24
|
|
|
} |
|
25
|
|
|
|
|
26
|
|
|
if err != nil { |
|
27
|
|
|
log.Failure("File read", fmt.Sprintf("%d row:", i), err.Error()) |
|
28
|
|
|
continue |
|
29
|
|
|
} |
|
30
|
|
|
|
|
31
|
|
|
if !columnsFound { |
|
32
|
|
|
columnsFound = true |
|
33
|
|
|
columns = readColumns(row, dict) |
|
34
|
|
|
|
|
35
|
|
|
logColumns(columns) |
|
36
|
|
|
continue |
|
37
|
|
|
} |
|
38
|
|
|
|
|
39
|
|
|
if skipLine(row) { |
|
40
|
|
|
continue |
|
41
|
|
|
} |
|
42
|
|
|
|
|
43
|
|
|
callback(row[0], mapPayload(columns, row, dict)) |
|
44
|
|
|
} |
|
45
|
|
|
} |
|
46
|
|
|
|
|
47
|
|
|
func readIntoMap(reader *csv.Reader) (map[int]string, error) { |
|
48
|
|
|
row, err := reader.Read() |
|
49
|
|
|
if err == nil || err == io.EOF { |
|
50
|
|
|
m := make(map[int]string, len(row)) |
|
51
|
|
|
for i, v := range row { |
|
52
|
|
|
m[i] = v |
|
53
|
|
|
} |
|
54
|
|
|
|
|
55
|
|
|
return m, err |
|
56
|
|
|
} |
|
57
|
|
|
|
|
58
|
|
|
return nil, err |
|
59
|
|
|
} |
|
60
|
|
|
|
|
61
|
|
|
func skipLine(line map[int]string) bool { |
|
62
|
|
|
return line == nil || len(line) == 0 || len(line[0]) == 0 |
|
63
|
|
|
} |
|
64
|
|
|
|
|
65
|
|
|
func logColumns(columns map[int]vars.Tag) { |
|
66
|
|
|
var cols []string |
|
67
|
|
|
for _, col := range columns { |
|
68
|
|
|
cols = append(cols, col.Key()) |
|
69
|
|
|
} |
|
70
|
|
|
|
|
71
|
|
|
log.Log("Parsed columns are:", cols...) |
|
72
|
|
|
} |
|
73
|
|
|
|
vvval /
go-metadata-scanner
