code.oscarkilo.com/klex-git

Hash:
02984c49e2e84a459548aa8ba0ec56c7bacf6071
Author:
Igor Naverniouk <[email protected]>
Date:
Wed Aug 28 12:45:45 2024 -0700
Message:
functions and pipelines in plan.go
diff --git a/plan/plan.go b/plan/plan.go
index 0d7fb88..5517795 100644
--- a/plan/plan.go
+++ b/plan/plan.go
@@ -6,6 +6,7 @@ import (
"log"
"path"
"sort"
+ "strings"

"oscarkilo.com/klex-git/api"
"oscarkilo.com/klex-git/config"
@@ -18,6 +19,8 @@ type Plan struct {
Commit *util.CommitInfo `json:"-"`
Client *api.Client `json:"-"`
ChangedDatasets []string `json:"changed_datasets"`
+ ChangedFunctions []string `json:"changed_functions"`
+ ChangedPipelines []string `json:"changed_pipelines"`
}

func NewPlan(cfg *config.Config, ci *util.CommitInfo, cl *api.Client) *Plan {
@@ -27,6 +30,8 @@ func NewPlan(cfg *config.Config, ci *util.CommitInfo, cl *api.Client) *Plan {
Client: cl,
}
p.findChangedDatasets()
+ p.findChangedFunctions()
+ p.findChangedPipelines()
return p
}

@@ -38,24 +43,69 @@ func (p *Plan) DebugString() string {
return string(b)
}

-func (p *Plan) findChangedDatasets() {
- datasets := make(map[string]bool)
+// findChangedFilesMatching returns the sorted list of changed files that:
+// - have the given path prefix
+// - have the given string suffix
+// - have been transformed by the given function
+// - aren't "" after the transformation
+func (p *Plan) findChangedFilesMatching(
+ pref, suff string,
+ transform func(string) string,
+) []string {
+ seen := make(map[string]bool)
for _, fpath := range p.Commit.ChangedPaths {
- if util.IsDir(fpath) {
- continue
- }
- if rest, ok := util.StripPathPrefix(fpath, p.Config.DatasetsDir); ok {
- dataset := path.Dir(rest)
- if dataset != "" {
- datasets[dataset] = true
+ if !util.IsDir(fpath) {
+ if rest, ok := util.StripPathPrefix(fpath, pref); ok {
+ if strings.HasSuffix(rest, suff) {
+ seen[transform(rest)] = true
+ }
}
}
}
- p.ChangedDatasets = make([]string, 0, len(datasets))
- for ds := range datasets {
- p.ChangedDatasets = append(p.ChangedDatasets, ds)
+ delete(seen, "") // in case it snuck in there
+ list := make([]string, 0, len(seen))
+ for f := range seen {
+ list = append(list, f)
+ }
+ sort.Strings(list)
+ return list
+}
+
+func (p *Plan) findChangedDatasets() {
+ if p.Config.DatasetsDir == "" {
+ return
+ }
+ p.ChangedDatasets = p.findChangedFilesMatching(
+ p.Config.DatasetsDir,
+ "",
+ path.Dir,
+ )
+}
+
+func (p *Plan) findChangedFunctions() {
+ if p.Config.FunctionsDir == "" {
+ return
+ }
+ p.ChangedFunctions = p.findChangedFilesMatching(
+ p.Config.FunctionsDir,
+ ".js",
+ func(f string) string {
+ return strings.TrimSuffix(f, ".js")
+ },
+ )
+}
+
+func (p *Plan) findChangedPipelines() {
+ if p.Config.PipelinesDir == "" {
+ return
}
- sort.Strings(p.ChangedDatasets)
+ p.ChangedPipelines = p.findChangedFilesMatching(
+ p.Config.PipelinesDir,
+ ".js",
+ func(f string) string {
+ return strings.TrimSuffix(f, ".js")
+ },
+ )
}

func (p *Plan) UploadDataset(ds string) error {
diff --git a/plan/plan_test.go b/plan/plan_test.go
index 5637817..9846950 100644
--- a/plan/plan_test.go
+++ b/plan/plan_test.go
@@ -8,7 +8,11 @@ import (
)

func TestNewPlan(t *testing.T) {
- cfg := &config.Config{DatasetsDir: "data"}
+ cfg := &config.Config{
+ DatasetsDir: "data",
+ FunctionsDir: "funcs",
+ PipelinesDir: "pipes",
+ }
commit := &util.CommitInfo{
ChangedPaths: []string{
"README.md",
@@ -16,11 +20,18 @@ func TestNewPlan(t *testing.T) {
"data/foo/1.txt",
"data/bar/0.txt",
"data/bee/bop/boo.txt",
+ "funcs/foo.js",
+ "funcs/bar/baz.js",
+ "funcs/README.md",
+ "pipes/README.md",
+ "pipes/qux.js",
},
}
p := NewPlan(cfg, commit, nil)
golden := &Plan{
ChangedDatasets: []string{"bar", "bee/bop", "foo"},
+ ChangedFunctions: []string{"bar/baz", "foo"},
+ ChangedPipelines: []string{"qux"},
}
if p.DebugString() != golden.DebugString() {
t.Errorf("NewPlan() = %s, want %s", p.DebugString(), golden.DebugString())
a/plan/plan.go
b/plan/plan.go
1
package plan
1
package plan
2
2
3
import (
3
import (
4
"encoding/json"
4
"encoding/json"
5
"fmt"
5
"fmt"
6
"log"
6
"log"
7
"path"
7
"path"
8
"sort"
8
"sort"
9
"strings"
9
10
10
"oscarkilo.com/klex-git/api"
11
"oscarkilo.com/klex-git/api"
11
"oscarkilo.com/klex-git/config"
12
"oscarkilo.com/klex-git/config"
12
"oscarkilo.com/klex-git/util"
13
"oscarkilo.com/klex-git/util"
13
)
14
)
14
15
15
// Plan is a list of steps that brings this repo in sync with Klex.
16
// Plan is a list of steps that brings this repo in sync with Klex.
16
type Plan struct {
17
type Plan struct {
17
Config *config.Config `json:"-"`
18
Config *config.Config `json:"-"`
18
Commit *util.CommitInfo `json:"-"`
19
Commit *util.CommitInfo `json:"-"`
19
Client *api.Client `json:"-"`
20
Client *api.Client `json:"-"`
20
ChangedDatasets []string `json:"changed_datasets"`
21
ChangedDatasets []string `json:"changed_datasets"`
22
ChangedFunctions []string `json:"changed_functions"`
23
ChangedPipelines []string `json:"changed_pipelines"`
21
}
24
}
22
25
23
func NewPlan(cfg *config.Config, ci *util.CommitInfo, cl *api.Client) *Plan {
26
func NewPlan(cfg *config.Config, ci *util.CommitInfo, cl *api.Client) *Plan {
24
p := &Plan{
27
p := &Plan{
25
Config: cfg,
28
Config: cfg,
26
Commit: ci,
29
Commit: ci,
27
Client: cl,
30
Client: cl,
28
}
31
}
29
p.findChangedDatasets()
32
p.findChangedDatasets()
33
p.findChangedFunctions()
34
p.findChangedPipelines()
30
return p
35
return p
31
}
36
}
32
37
33
func (p *Plan) DebugString() string {
38
func (p *Plan) DebugString() string {
34
b, err := json.MarshalIndent(p, "", " ")
39
b, err := json.MarshalIndent(p, "", " ")
35
if err != nil {
40
if err != nil {
36
return fmt.Sprintf("JSON error: %v", err)
41
return fmt.Sprintf("JSON error: %v", err)
37
}
42
}
38
return string(b)
43
return string(b)
39
}
44
}
40
45
41
func (p *Plan) findChangedDatasets() {
46
// findChangedFilesMatching returns the sorted list of changed files that:
42
datasets := make(map[string]bool)
47
// - have the given path prefix
48
// - have the given string suffix
49
// - have been transformed by the given function
50
// - aren't "" after the transformation
51
func (p *Plan) findChangedFilesMatching(
52
pref, suff string,
53
transform func(string) string,
54
) []string {
55
seen := make(map[string]bool)
43
for _, fpath := range p.Commit.ChangedPaths {
56
for _, fpath := range p.Commit.ChangedPaths {
44
if util.IsDir(fpath) {
57
if !util.IsDir(fpath) {
45
continue
58
if rest, ok := util.StripPathPrefix(fpath, pref); ok {
46
}
59
if strings.HasSuffix(rest, suff) {
47
if rest, ok := util.StripPathPrefix(fpath, p.Config.DatasetsDir); ok {
60
seen[transform(rest)] = true
48
dataset := path.Dir(rest)
61
}
49
if dataset != "" {
50
datasets[dataset] = true
51
}
62
}
52
}
63
}
53
}
64
}
54
p.ChangedDatasets = make([]string, 0, len(datasets))
65
delete(seen, "") // in case it snuck in there
55
for ds := range datasets {
66
list := make([]string, 0, len(seen))
56
p.ChangedDatasets = append(p.ChangedDatasets, ds)
67
for f := range seen {
68
list = append(list, f)
69
}
70
sort.Strings(list)
71
return list
72
}
73
74
func (p *Plan) findChangedDatasets() {
75
if p.Config.DatasetsDir == "" {
76
return
77
}
78
p.ChangedDatasets = p.findChangedFilesMatching(
79
p.Config.DatasetsDir,
80
"",
81
path.Dir,
82
)
83
}
84
85
func (p *Plan) findChangedFunctions() {
86
if p.Config.FunctionsDir == "" {
87
return
88
}
89
p.ChangedFunctions = p.findChangedFilesMatching(
90
p.Config.FunctionsDir,
91
".js",
92
func(f string) string {
93
return strings.TrimSuffix(f, ".js")
94
},
95
)
96
}
97
98
func (p *Plan) findChangedPipelines() {
99
if p.Config.PipelinesDir == "" {
100
return
57
}
101
}
58
sort.Strings(p.ChangedDatasets)
102
p.ChangedPipelines = p.findChangedFilesMatching(
103
p.Config.PipelinesDir,
104
".js",
105
func(f string) string {
106
return strings.TrimSuffix(f, ".js")
107
},
108
)
59
}
109
}
60
110
61
func (p *Plan) UploadDataset(ds string) error {
111
func (p *Plan) UploadDataset(ds string) error {
62
from := path.Join(p.Config.DatasetsDir, ds)
112
from := path.Join(p.Config.DatasetsDir, ds)
63
to := path.Join(p.Config.ProjectName, ds)
113
to := path.Join(p.Config.ProjectName, ds)
64
log.Printf("reading dataset %s", ds)
114
log.Printf("reading dataset %s", ds)
65
data, err := util.ReadDir(from)
115
data, err := util.ReadDir(from)
66
if err != nil {
116
if err != nil {
67
return fmt.Errorf("Error reading directory %s: %v", from, err)
117
return fmt.Errorf("Error reading directory %s: %v", from, err)
68
}
118
}
69
log.Printf("uploading dataset %s", ds)
119
log.Printf("uploading dataset %s", ds)
70
return p.Client.NewDataset(to, data)
120
return p.Client.NewDataset(to, data)
71
}
121
}
a/plan/plan_test.go
b/plan/plan_test.go
1
package plan
1
package plan
2
2
3
import (
3
import (
4
"testing"
4
"testing"
5
5
6
"oscarkilo.com/klex-git/config"
6
"oscarkilo.com/klex-git/config"
7
"oscarkilo.com/klex-git/util"
7
"oscarkilo.com/klex-git/util"
8
)
8
)
9
9
10
func TestNewPlan(t *testing.T) {
10
func TestNewPlan(t *testing.T) {
11
cfg := &config.Config{DatasetsDir: "data"}
11
cfg := &config.Config{
12
DatasetsDir: "data",
13
FunctionsDir: "funcs",
14
PipelinesDir: "pipes",
15
}
12
commit := &util.CommitInfo{
16
commit := &util.CommitInfo{
13
ChangedPaths: []string{
17
ChangedPaths: []string{
14
"README.md",
18
"README.md",
15
"data/foo/0.txt",
19
"data/foo/0.txt",
16
"data/foo/1.txt",
20
"data/foo/1.txt",
17
"data/bar/0.txt",
21
"data/bar/0.txt",
18
"data/bee/bop/boo.txt",
22
"data/bee/bop/boo.txt",
23
"funcs/foo.js",
24
"funcs/bar/baz.js",
25
"funcs/README.md",
26
"pipes/README.md",
27
"pipes/qux.js",
19
},
28
},
20
}
29
}
21
p := NewPlan(cfg, commit, nil)
30
p := NewPlan(cfg, commit, nil)
22
golden := &Plan{
31
golden := &Plan{
23
ChangedDatasets: []string{"bar", "bee/bop", "foo"},
32
ChangedDatasets: []string{"bar", "bee/bop", "foo"},
33
ChangedFunctions: []string{"bar/baz", "foo"},
34
ChangedPipelines: []string{"qux"},
24
}
35
}
25
if p.DebugString() != golden.DebugString() {
36
if p.DebugString() != golden.DebugString() {
26
t.Errorf("NewPlan() = %s, want %s", p.DebugString(), golden.DebugString())
37
t.Errorf("NewPlan() = %s, want %s", p.DebugString(), golden.DebugString())
27
}
38
}
28
}
39
}