Hash:
f8416ee9defcad6e687f5293320b77a81aeaed57
Author:
Igor Naverniouk <[email protected]>
Date:
Mon Sep 23 10:03:23 2024 -0700
Message:
diff --git a/api/datasets.go b/api/datasets.go
index 71edb9b..115f42e 100644
--- a/api/datasets.go
+++ b/api/datasets.go
@@ -48,3 +48,29 @@ type GetDatasetResponse struct {
DatasetVersion DatasetVersion `json:"dataset_version"`
Records []KV `json:"records"`
}
+
+// BeginNewDatasetRequest is for the new API for uploading datasets.
+// It's meant to work for huge datasets as well as small ones.
+// The flow goes like this:
+// 1. Call /klex/datasets/begin_new and get a version key.
+// 2. Repeatedly call /klex/datasets/upload_kv with chunks of data.
+// 3. Call /klex/datasets/end_new to commit and name the dataset.
+type BeginNewDatasetRequest struct {
+ Name string `json:"name"`
+}
+
+type BeginNewDatasetResponse struct {
+ VersionKey string `json:"version_key"`
+}
+
+// UploadKVRequest is for uploading some of the kv-pairs of a new dataset.
+type UploadKVRequest struct {
+ VersionKey string `json:"version_key"`
+ Records []KV `json:"records"`
+}
+
+type EndNewDatasetsRequest struct {
+ Name string `json:"name"`
+ VersionKey string `json:"version_key"`
+ Size int `json:"size"`
+}
index 71edb9b..115f42e 100644
--- a/api/datasets.go
+++ b/api/datasets.go
@@ -48,3 +48,29 @@ type GetDatasetResponse struct {
DatasetVersion DatasetVersion `json:"dataset_version"`
Records []KV `json:"records"`
}
+
+// BeginNewDatasetRequest is for the new API for uploading datasets.
+// It's meant to work for huge datasets as well as small ones.
+// The flow goes like this:
+// 1. Call /klex/datasets/begin_new and get a version key.
+// 2. Repeatedly call /klex/datasets/upload_kv with chunks of data.
+// 3. Call /klex/datasets/end_new to commit and name the dataset.
+type BeginNewDatasetRequest struct {
+ Name string `json:"name"`
+}
+
+type BeginNewDatasetResponse struct {
+ VersionKey string `json:"version_key"`
+}
+
+// UploadKVRequest is for uploading some of the kv-pairs of a new dataset.
+type UploadKVRequest struct {
+ VersionKey string `json:"version_key"`
+ Records []KV `json:"records"`
+}
+
+type EndNewDatasetsRequest struct {
+ Name string `json:"name"`
+ VersionKey string `json:"version_key"`
+ Size int `json:"size"`
+}
a/api/datasets.go
b/api/datasets.go
1
package api
1
package api
2
2
3
type NewDatasetRequest struct {
3
type NewDatasetRequest struct {
4
Name string `json:"name"` // no need to be unique
4
Name string `json:"name"` // no need to be unique
5
Data []string `json:"data"`
5
Data []string `json:"data"`
6
}
6
}
7
7
8
type NewDatasetResponse struct {
8
type NewDatasetResponse struct {
9
Name string `json:"name"`
9
Name string `json:"name"`
10
DateCreated string `json:"date_created"` // RFC8601 with millis
10
DateCreated string `json:"date_created"` // RFC8601 with millis
11
Size int `json:"size"`
11
Size int `json:"size"`
12
}
12
}
13
13
14
type RenameDatasetRequest struct {
14
type RenameDatasetRequest struct {
15
OldName string `json:"old_name"`
15
OldName string `json:"old_name"`
16
NewName string `json:"new_name"`
16
NewName string `json:"new_name"`
17
}
17
}
18
18
19
type DeleteDatasetRequest struct {
19
type DeleteDatasetRequest struct {
20
Name string `json:"name"`
20
Name string `json:"name"`
21
}
21
}
22
22
23
type DatasetVersion struct {
23
type DatasetVersion struct {
24
Key string `json:"key"`
24
Key string `json:"key"`
25
Size int `json:"size"`
25
Size int `json:"size"`
26
Date string `json:"date"`
26
Date string `json:"date"`
27
}
27
}
28
28
29
type Dataset struct {
29
type Dataset struct {
30
Name string `json:"name"`
30
Name string `json:"name"`
31
OwnerOwid string `json:"owner_owid"`
31
OwnerOwid string `json:"owner_owid"`
32
OwnerUsername string `json:"owner_username"`
32
OwnerUsername string `json:"owner_username"`
33
ReaderOwid string `json:"reader_owid"`
33
ReaderOwid string `json:"reader_owid"`
34
ReaderUsername string `json:"reader_username"`
34
ReaderUsername string `json:"reader_username"`
35
Versions []DatasetVersion `json:"versions"`
35
Versions []DatasetVersion `json:"versions"`
36
}
36
}
37
37
38
type ListDatasetsResponse struct {
38
type ListDatasetsResponse struct {
39
Datasets []Dataset `json:"datasets"`
39
Datasets []Dataset `json:"datasets"`
40
}
40
}
41
41
42
type KV struct {
42
type KV struct {
43
Key string `json:"key"`
43
Key string `json:"key"`
44
Value string `json:"value"`
44
Value string `json:"value"`
45
}
45
}
46
46
47
type GetDatasetResponse struct {
47
type GetDatasetResponse struct {
48
DatasetVersion DatasetVersion `json:"dataset_version"`
48
DatasetVersion DatasetVersion `json:"dataset_version"`
49
Records []KV `json:"records"`
49
Records []KV `json:"records"`
50
}
50
}
51
52
// BeginNewDatasetRequest is for the new API for uploading datasets.
53
// It's meant to work for huge datasets as well as small ones.
54
// The flow goes like this:
55
// 1. Call /klex/datasets/begin_new and get a version key.
56
// 2. Repeatedly call /klex/datasets/upload_kv with chunks of data.
57
// 3. Call /klex/datasets/end_new to commit and name the dataset.
58
type BeginNewDatasetRequest struct {
59
Name string `json:"name"`
60
}
61
62
type BeginNewDatasetResponse struct {
63
VersionKey string `json:"version_key"`
64
}
65
66
// UploadKVRequest is for uploading some of the kv-pairs of a new dataset.
67
type UploadKVRequest struct {
68
VersionKey string `json:"version_key"`
69
Records []KV `json:"records"`
70
}
71
72
type EndNewDatasetsRequest struct {
73
Name string `json:"name"`
74
VersionKey string `json:"version_key"`
75
Size int `json:"size"`
76
}