code.oscarkilo.com/klex-git/api/pipelines.go

package api

type PipelineVersion struct {
  Hash string `json:"hash"`
  Date string `json:"date"` // RFC8601 with millis
  JS string `json:"js"`
}

type Pipeline struct {
  Name string `json:"name"`
  Versions []PipelineVersion `json:"versions"`
}

type NewPipelineRequest struct {
  Name string `json:"name"`  // no need to be unique
  JsCode string `json:"js_code"`
}

type NewPipelineResponse struct {
  Name string `json:"name"`
  DateCreated string `json:"date_created"`  // RFC8601 with millis
}

type DeletePipelineRequest struct {
  Name string `json:"name"`
}

type DeletePipelineResponse struct {
}

type ListPipelinesResponse struct {
  Pipes []Pipeline `json:"pipes"`
}

type NameOrHash struct {
  Name string `json:"name"`  // Func, dataset, or pipeline name
  Hash string `json:"hash"`  // Func, dataset, or pipeline version hash
  Suff string `json:"suff"`  // Dataset suffix, within a pipeline
}

type PipelineExecStage struct {
  // map
  MapFunc NameOrHash `json:"map_func"`
  MapIns []NameOrHash `json:"map_ins"`
  MapOut NameOrHash `json:"map_out"`

  // TODO: shuffle?
  // TODO: reduce
}

type StageProgress struct {
  InSizes []int `json:"in_sizes"`
  InConsumed []int `json:"in_consumed"`
  NextKey string `json:"next_key"`
  OutEmitted int `json:"out_emitted"`
  OutputNamed bool `json:"output_named"`
  IsDoneWriting bool `json:"is_done_writing"`
  IsDone bool `json:"is_done"`
}

type PipelineExecPlan struct {
  Stages []PipelineExecStage `json:"stages"`
}

type PlanProgress struct {
  Stages []StageProgress `json:"stages"`
  Status string `json:"status"`
  Errors []string `json:"errors"`
}

type PipelineRun struct {
  Key string `json:"key"`
  PipelineName string `json:"pipeline_name"`
  PipelineVersion string `json:"pipeline_version"`
  Plan PipelineExecPlan `json:"plan"`
  StartTime string `json:"start_time"`  // RFC8601 with millis
  Progress PlanProgress `json:"progress"`
}

type LaunchPipelineRequest struct {
  Name string `json:"name"`
}

type LaunchPipelineResponse struct {
  Run PipelineRun `json:"run"`
}

type ListRunsResponse struct {
  Runs []PipelineRun `json:"runs"`
}

type GetRunResponse struct {
  PipelineVersion PipelineVersion `json:"pipeline_version"`
  Run PipelineRun `json:"run"`
}