code.oscarkilo.com/klex-git #a5cfc88dc3aaaf41775431d86d4ab0981235aa8b

Hash:

a5cfc88dc3aaaf41775431d86d4ab0981235aa8b

Author:

Date:

Wed May 13 13:52:15 2026 -0400

Message:

Rename ContentBlock Type:"image" to "document"; add CanSeePDFs Klex's content-block wire format now uses a single Type:"document" for both images and PDFs, disambiguated by Source.MediaType. The old Type:"image" is gone. LLMFunc grows a new CanSeePDFs flag (json: can_see_pdfs) alongside the existing CanSeeImages. The two are independent because Fireworks/Ollama vision models accept images but not PDFs. Stored func versions without the new key unmarshal to CanSeePDFs=false, so no data migration is needed; CanSeeImages stays exactly as it was on disk. Callers in //klex-git/exemplary and //klex-git/one now emit Type:"document". Downstream //funky and //ithfm follow in separate commits. New tests in api/ pin both the document-block JSON shape and the images/PDFs flag combinations.

diff --git a/api/funcs.go b/api/funcs.go
index bc2cf69..beb5ae0 100644
--- a/api/funcs.go
+++ b/api/funcs.go
@@ -42,7 +42,18 @@ type LLMFunc struct {
// Model is the provider-assigned name of the LLM.
Model string `json:"model"`

+ // CanSeeImages is true iff the model accepts image attachments
+ // (PNG/JPEG/GIF/WEBP). Production func versions already use
+ // this flag — do not rename.
CanSeeImages bool `json:"can_see_images"`
+
+ // CanSeePDFs is true iff the model accepts PDF attachments.
+ // Independent from CanSeeImages because some providers (e.g.
+ // Fireworks, Ollama) support images but not PDFs. Defaults to
+ // false; stored func versions without the key get false on
+ // unmarshal.
+ CanSeePDFs bool `json:"can_see_pdfs"`
+
CanHaveSystemPromps bool `json:"can_have_system_prompts"`
CanUseTools bool `json:"can_use_tools"`
CanStream bool `json:"can_stream"`
diff --git a/api/funcs_test.go b/api/funcs_test.go
new file mode 100644
index 0000000..160391d
--- /dev/null
+++ b/api/funcs_test.go
@@ -0,0 +1,114 @@
+package api
+
+// These tests pin the wire format of LLMFunc. CanSeeImages is the
+// pre-existing capability flag (do not rename — production func
+// definitions at oscarkilo.com/klex/funcs already use it).
+// CanSeePDFs is the new capability flag for PDF attachments. The
+// two are independent because some providers support images but
+// not PDFs (Fireworks, Ollama, xAI-via-v1-of-our-adapter).
+
+import "encoding/json"
+import "strings"
+import "testing"
+
+func TestLLMFuncMarshalImagesPDFs(t *testing.T) {
+ f := LLMFunc{
+ Provider: "anthropic",
+ Model: "claude-sonnet-4-6",
+ CanSeeImages: true,
+ CanSeePDFs: true,
+ CanStream: true,
+ }
+ buf, err := json.Marshal(f)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ s := string(buf)
+ if !strings.Contains(s, `"can_see_images":true`) {
+ t.Errorf("missing can_see_images in %s", s)
+ }
+ if !strings.Contains(s, `"can_see_pdfs":true`) {
+ t.Errorf("missing can_see_pdfs in %s", s)
+ }
+}
+
+func TestLLMFuncUnmarshalLegacyData(t *testing.T) {
+ // Existing production func versions are stored without a
+ // can_see_pdfs key. They must unmarshal to CanSeePDFs=false
+ // without error.
+ in := []byte(
+ `{"provider":"anthropic","model":"x","can_see_images":true}`)
+ var f LLMFunc
+ if err := json.Unmarshal(in, &f); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if !f.CanSeeImages {
+ t.Errorf("CanSeeImages = false, want true")
+ }
+ if f.CanSeePDFs {
+ t.Errorf("CanSeePDFs = true, want false (key absent)")
+ }
+}
+
+func TestLLMFuncUnmarshalPDFsOnly(t *testing.T) {
+ // A model that accepts PDFs but not images (rare but possible
+ // if a future provider goes that way).
+ in := []byte(
+ `{"provider":"x","model":"y","can_see_pdfs":true}`)
+ var f LLMFunc
+ if err := json.Unmarshal(in, &f); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if f.CanSeeImages {
+ t.Errorf("CanSeeImages = true, want false")
+ }
+ if !f.CanSeePDFs {
+ t.Errorf("CanSeePDFs = false, want true")
+ }
+}
+
+func TestLLMFuncUnmarshalBoth(t *testing.T) {
+ in := []byte(
+ `{"provider":"openai","model":"gpt-4o",` +
+ `"can_see_images":true,"can_see_pdfs":true}`)
+ var f LLMFunc
+ if err := json.Unmarshal(in, &f); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if !f.CanSeeImages || !f.CanSeePDFs {
+ t.Errorf("expected both flags true; got %+v", f)
+ }
+}
+
+func TestLLMFuncUnmarshalNeither(t *testing.T) {
+ in := []byte(`{"provider":"x","model":"y"}`)
+ var f LLMFunc
+ if err := json.Unmarshal(in, &f); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if f.CanSeeImages || f.CanSeePDFs {
+ t.Errorf("expected both flags false; got %+v", f)
+ }
+}
+
+func TestLLMFuncRoundTrip(t *testing.T) {
+ orig := LLMFunc{
+ Provider: "openai",
+ Model: "gpt-4o",
+ CanSeeImages: true,
+ CanSeePDFs: true,
+ CanUseTools: true,
+ CanStream: true,
+ }
+ buf, err := json.Marshal(orig)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ var got LLMFunc
+ if err := json.Unmarshal(buf, &got); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if got != orig {
+ t.Errorf("round-trip diff\nwant: %+v\n got: %+v", orig, got)
+ }
+}
diff --git a/api/messages.go b/api/messages.go
index 9e4c896..534e518 100644
--- a/api/messages.go
+++ b/api/messages.go
@@ -16,13 +16,14 @@ type MessagesRequest struct {
}

type ContentBlock struct {
- // Type is "text", "image", "tool_use", or "tool_result".
+ // Type is "text", "document", "tool_use", or "tool_result".
Type string `json:"type"`

// Text is for Type="text"
Text string `json:"text,omitempty"`

- // Source is for Type="image"
+ // Source is for Type="document". MediaType disambiguates
+ // between image attachments and PDF attachments.
Source *ContentSource `json:"source,omitempty"`

// ID, Name, and Input are for Type="tool_use".
@@ -44,8 +45,13 @@ type ContentSource struct {
// MediaType can be one of:
// - "image/jpeg",
// - "image/png",
- // - "image/gif", or
- // - "image/webp".
+ // - "image/gif",
+ // - "image/webp", or
+ // - "application/pdf".
+ // Whether a particular MIME is accepted depends on the
+ // backing model; the func registry's CanSeeDocuments
+ // capability gates attachments at all but does not split
+ // by MIME.
MediaType string `json:"media_type,omitempty"`

Data string `json:"data,omitempty"`
diff --git a/api/messages_test.go b/api/messages_test.go
new file mode 100644
index 0000000..da41ea3
--- /dev/null
+++ b/api/messages_test.go
@@ -0,0 +1,100 @@
+package api
+
+// These tests pin the wire format of ContentBlock so callers across
+// //funky, //ithfm, and other consumers can rely on stable JSON
+// shapes. They fail loudly if the JSON tags or field names ever
+// drift.
+
+import "encoding/json"
+import "strings"
+import "testing"
+
+func TestDocumentBlockMarshal(t *testing.T) {
+ block := ContentBlock{
+ Type: "document",
+ Source: &ContentSource{
+ Type: "base64",
+ MediaType: "application/pdf",
+ Data: "JVBERi0xLjQK",
+ },
+ }
+ buf, err := json.Marshal(block)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ want := `{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0xLjQK"}}`
+ if string(buf) != want {
+ t.Errorf(
+ "marshal mismatch\nwant: %s\n got: %s", want, buf)
+ }
+}
+
+func TestDocumentBlockUnmarshal(t *testing.T) {
+ in := []byte(
+ `{"type":"document","source":{"type":"base64",` +
+ `"media_type":"application/pdf","data":"JVBERi0xLjQK"}}`)
+ var block ContentBlock
+ if err := json.Unmarshal(in, &block); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if block.Type != "document" {
+ t.Errorf("Type = %q, want %q", block.Type, "document")
+ }
+ if block.Source == nil {
+ t.Fatalf("Source is nil")
+ }
+ if block.Source.MediaType != "application/pdf" {
+ t.Errorf(
+ "MediaType = %q, want application/pdf",
+ block.Source.MediaType)
+ }
+ if block.Source.Data != "JVBERi0xLjQK" {
+ t.Errorf("Data = %q", block.Source.Data)
+ }
+}
+
+func TestDocumentBlockImageMimeRoundTrip(t *testing.T) {
+ // A "document" with an image/* MIME is how the wire format
+ // carries what used to live under Type:"image".
+ block := ContentBlock{
+ Type: "document",
+ Source: &ContentSource{
+ Type: "base64",
+ MediaType: "image/png",
+ Data: "iVBORw0KGgo=",
+ },
+ }
+ buf, err := json.Marshal(block)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ if !strings.Contains(string(buf), `"type":"document"`) {
+ t.Errorf("expected type:document, got %s", buf)
+ }
+ if !strings.Contains(string(buf), `"media_type":"image/png"`) {
+ t.Errorf("expected media_type image/png, got %s", buf)
+ }
+ var got ContentBlock
+ if err := json.Unmarshal(buf, &got); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if got.Type != "document" {
+ t.Errorf("Type = %q", got.Type)
+ }
+ if got.Source == nil ||
+ got.Source.MediaType != "image/png" {
+ t.Errorf("Source mismatch: %+v", got.Source)
+ }
+}
+
+func TestTextBlockUnchanged(t *testing.T) {
+ // Documents the unchanged text-block wire format.
+ block := ContentBlock{Type: "text", Text: "hello"}
+ buf, err := json.Marshal(block)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ if string(buf) != `{"type":"text","text":"hello"}` {
+ t.Errorf("unexpected text-block JSON: %s", buf)
+ }
+}
diff --git a/exemplary/main.go b/exemplary/main.go
index 6b4720f..90f1574 100644
--- a/exemplary/main.go
+++ b/exemplary/main.go
@@ -159,7 +159,7 @@ func main() {
case "jpg", "jpeg", "png", "webp":
bytes := readFile(c.Name + "." + suffix)
user.Content = append(user.Content, api.ContentBlock{
- Type: "image",
+ Type: "document",
Source: &api.ContentSource{
Type: "base64",
MediaType: http.DetectContentType(bytes),
diff --git a/one/main.go b/one/main.go
index 4326dac..a7a989b 100644
--- a/one/main.go
+++ b/one/main.go
@@ -84,7 +84,7 @@ func main() {
log.Fatalf("Unsupported image type: %s", mime_type)
}
msg.Content = append(msg.Content, api.ContentBlock{
- Type: "image",
+ Type: "document",
Source: &api.ContentSource{
Type: "base64",
MediaType: mime_type,

package api

type NewFuncRequest struct {

Name string `json:"name"` // no need to be unique

JsCode string `json:"js_code,omitempty"` // LLM2 must be nil

LLM2 *LLMFunc `json:"llm2,omitempty"` // JSCode must be ""

}

type NewFuncResponse struct {

Name string `json:"name"`

DateCreated string `json:"date_created"` // RFC8601 with millis

}

type DeleteFuncRequest struct {

Name string `json:"name"`

}

type DeleteFuncResponse struct {

}

type FuncVersion struct {

// Hash is the globally unique ID of this immutable object.

Hash string `json:"hash"`

// Date is the time of creation, according to the server's clock.

Date string `json:"date"`

// JS is JavaScript code that implements this function.

JS string `json:"js,omitempty"`

// LLM is the old "provider:model" string. Deprecated.

LLM string `json:"llm,omitempty"`

// LLM2 confugures where an how the LLM is run.

LLM2 *LLMFunc `json:"llm2,omitempty"`

}

type LLMFunc struct {

// Provider is "openai", "anthropic", "fireworks", etc.

Provider string `json:"provider"`

// Model is the provider-assigned name of the LLM.

Model string `json:"model"`

// CanSeeImages is true iff the model accepts image attachments

// (PNG/JPEG/GIF/WEBP). Production func versions already use

// this flag — do not rename.

CanSeeImages bool `json:"can_see_images"`

// CanSeePDFs is true iff the model accepts PDF attachments.

// Independent from CanSeeImages because some providers (e.g.

// Fireworks, Ollama) support images but not PDFs. Defaults to

// false; stored func versions without the key get false on

// unmarshal.

CanSeePDFs bool `json:"can_see_pdfs"`

CanHaveSystemPromps bool `json:"can_have_system_prompts"`

CanUseTools bool `json:"can_use_tools"`

CanStream bool `json:"can_stream"`

}

type Func struct {

Name string `json:"name"`

Versions []FuncVersion `json:"versions"`

}

type ListFuncsResponse struct {

Funcs []Func `json:"funcs"`

}

package api

// These tests pin the wire format of LLMFunc. CanSeeImages is the

// pre-existing capability flag (do not rename — production func

// definitions at oscarkilo.com/klex/funcs already use it).

// CanSeePDFs is the new capability flag for PDF attachments. The

// two are independent because some providers support images but

// not PDFs (Fireworks, Ollama, xAI-via-v1-of-our-adapter).

import "encoding/json"

import "strings"

import "testing"

func TestLLMFuncMarshalImagesPDFs(t *testing.T) {

f := LLMFunc{

Provider: "anthropic",

Model: "claude-sonnet-4-6",

CanSeeImages: true,

CanSeePDFs: true,

CanStream: true,

}

buf, err := json.Marshal(f)

if err != nil {

t.Fatalf("marshal: %v", err)

}

s := string(buf)

if !strings.Contains(s, `"can_see_images":true`) {

t.Errorf("missing can_see_images in %s", s)

}

if !strings.Contains(s, `"can_see_pdfs":true`) {

t.Errorf("missing can_see_pdfs in %s", s)

}

func TestLLMFuncUnmarshalLegacyData(t *testing.T) {

// Existing production func versions are stored without a

// can_see_pdfs key. They must unmarshal to CanSeePDFs=false

// without error.

in := []byte(

`{"provider":"anthropic","model":"x","can_see_images":true}`)

var f LLMFunc

if err := json.Unmarshal(in, &f); err != nil {

t.Fatalf("unmarshal: %v", err)

}

if !f.CanSeeImages {

t.Errorf("CanSeeImages = false, want true")

}

if f.CanSeePDFs {

t.Errorf("CanSeePDFs = true, want false (key absent)")

}

func TestLLMFuncUnmarshalPDFsOnly(t *testing.T) {

// A model that accepts PDFs but not images (rare but possible

// if a future provider goes that way).

in := []byte(

`{"provider":"x","model":"y","can_see_pdfs":true}`)

var f LLMFunc

if err := json.Unmarshal(in, &f); err != nil {

t.Fatalf("unmarshal: %v", err)

}

if f.CanSeeImages {

t.Errorf("CanSeeImages = true, want false")

}

if !f.CanSeePDFs {

t.Errorf("CanSeePDFs = false, want true")

}

func TestLLMFuncUnmarshalBoth(t *testing.T) {

in := []byte(

`{"provider":"openai","model":"gpt-4o",` +

`"can_see_images":true,"can_see_pdfs":true}`)

var f LLMFunc

if err := json.Unmarshal(in, &f); err != nil {

t.Fatalf("unmarshal: %v", err)

}

if !f.CanSeeImages || !f.CanSeePDFs {

t.Errorf("expected both flags true; got %+v", f)

}

func TestLLMFuncUnmarshalNeither(t *testing.T) {

in := []byte(`{"provider":"x","model":"y"}`)

var f LLMFunc

if err := json.Unmarshal(in, &f); err != nil {

t.Fatalf("unmarshal: %v", err)

}

if f.CanSeeImages || f.CanSeePDFs {

t.Errorf("expected both flags false; got %+v", f)

}

func TestLLMFuncRoundTrip(t *testing.T) {

orig := LLMFunc{

Provider: "openai",

Model: "gpt-4o",

CanSeeImages: true,

CanSeePDFs: true,

100

CanUseTools: true,

101

CanStream: true,

102

}

103

buf, err := json.Marshal(orig)

104

if err != nil {

105

t.Fatalf("marshal: %v", err)

106

}

107

var got LLMFunc

108

if err := json.Unmarshal(buf, &got); err != nil {

109

t.Fatalf("unmarshal: %v", err)

110

}

111

if got != orig {

112

t.Errorf("round-trip diff\nwant: %+v\n got: %+v", orig, got)

113

}

114

}

package api

type ChatMessage struct {

Role string `json:"role"`

Content []ContentBlock `json:"content"`

}

// MessageRequest is generalizes across OpenAI, Anthropic, etc.

type MessagesRequest struct {

Model string `json:"model"`

Messages []ChatMessage `json:"messages"`

MaxTokens int `json:"max_tokens"`

System string `json:"system,omitempty"`

Temperature float64 `json:"temperature"`

Tools []Tool `json:"tools,omitempty"`

}

type ContentBlock struct {

// Type is "text", "image", "tool_use", or "tool_result".

// Type is "text", "document", "tool_use", or "tool_result".

Type string `json:"type"`

// Text is for Type="text"

Text string `json:"text,omitempty"`

// Source is for Type="image"

// Source is for Type="document". MediaType disambiguates

// between image attachments and PDF attachments.

Source *ContentSource `json:"source,omitempty"`

// ID, Name, and Input are for Type="tool_use".

ID string `json:"id,omitempty"`

Name string `json:"name,omitempty"`

Input interface{} `json:"input,omitempty"`

// ToolUseID, Content, and Output are for

// Type="tool_result".

ToolUseID string `json:"tool_use_id,omitempty"`

Content string `json:"content,omitempty"`

Output string `json:"output,omitempty"`

}

type ContentSource struct {

// Type can only be "base64".

Type string `json:"type"`

// MediaType can be one of:

// - "image/jpeg",

// - "image/png",

// - "image/gif", or

// - "image/gif",

// - "image/webp".

// - "image/webp", or

// - "application/pdf".

// Whether a particular MIME is accepted depends on the

// backing model; the func registry's CanSeeDocuments

// capability gates attachments at all but does not split

// by MIME.

MediaType string `json:"media_type,omitempty"`

Data string `json:"data,omitempty"`

}

type Usage struct {

InputTokens int `json:"input_tokens,omitempty"`

CacheCreationInputTokens *int `json:"cache_creation_input_tokens,omitempty"`

CacheReadInputTokens *int `json:"cache_read_input_tokens,omitempty"`

OutputTokens int `json:"output_tokens,omitempty"`

}

type ErrorResponse struct {

Type string `json:"type"`

Message string `json:"message"`

}

type MessagesResponse struct {

Id string `json:"id"`

Type string `json:"type"`

Role string `json:"role"`

Content []ContentBlock `json:"content"`

Model string `json:"model"`

StopReason *string `json:"stop_reason,omitempty"`

StopSequence *string `json:"stop_sequence,omitempty"`

Usage Usage `json:"usage"`

Error *ErrorResponse `json:"error,omitempty"`

}

type Tool struct {

Type string `json:"type"`

Function *ToolFunction `json:"function"`

}

type ToolFunction struct {

Name string `json:"name"`

Description string `json:"description"`

InputSchema interface{} `json:"input_schema"`

}

package api

// These tests pin the wire format of ContentBlock so callers across

// //funky, //ithfm, and other consumers can rely on stable JSON

// shapes. They fail loudly if the JSON tags or field names ever

// drift.

import "encoding/json"

import "strings"

import "testing"

func TestDocumentBlockMarshal(t *testing.T) {

block := ContentBlock{

Type: "document",

Source: &ContentSource{

Type: "base64",

MediaType: "application/pdf",

Data: "JVBERi0xLjQK",

}

buf, err := json.Marshal(block)

if err != nil {

t.Fatalf("marshal: %v", err)

}

want := `{"type":"document","source":{"type":"base64","media_type":"application/pdf","data":"JVBERi0xLjQK"}}`

if string(buf) != want {

t.Errorf(

"marshal mismatch\nwant: %s\n got: %s", want, buf)

}

func TestDocumentBlockUnmarshal(t *testing.T) {

in := []byte(

`{"type":"document","source":{"type":"base64",` +

`"media_type":"application/pdf","data":"JVBERi0xLjQK"}}`)

var block ContentBlock

if err := json.Unmarshal(in, &block); err != nil {

t.Fatalf("unmarshal: %v", err)

}

if block.Type != "document" {

t.Errorf("Type = %q, want %q", block.Type, "document")

}

if block.Source == nil {

t.Fatalf("Source is nil")

}

if block.Source.MediaType != "application/pdf" {

t.Errorf(

"MediaType = %q, want application/pdf",

block.Source.MediaType)

}

if block.Source.Data != "JVBERi0xLjQK" {

t.Errorf("Data = %q", block.Source.Data)

}

func TestDocumentBlockImageMimeRoundTrip(t *testing.T) {

// A "document" with an image/* MIME is how the wire format

// carries what used to live under Type:"image".

block := ContentBlock{

Type: "document",

Source: &ContentSource{

Type: "base64",

MediaType: "image/png",

Data: "iVBORw0KGgo=",

}

buf, err := json.Marshal(block)

if err != nil {

t.Fatalf("marshal: %v", err)

}

if !strings.Contains(string(buf), `"type":"document"`) {

t.Errorf("expected type:document, got %s", buf)

}

if !strings.Contains(string(buf), `"media_type":"image/png"`) {

t.Errorf("expected media_type image/png, got %s", buf)

}

var got ContentBlock

if err := json.Unmarshal(buf, &got); err != nil {

t.Fatalf("unmarshal: %v", err)

}

if got.Type != "document" {

t.Errorf("Type = %q", got.Type)

}

if got.Source == nil ||

got.Source.MediaType != "image/png" {

t.Errorf("Source mismatch: %+v", got.Source)

}

func TestTextBlockUnchanged(t *testing.T) {

// Documents the unchanged text-block wire format.

block := ContentBlock{Type: "text", Text: "hello"}

buf, err := json.Marshal(block)

if err != nil {

t.Fatalf("marshal: %v", err)

}

if string(buf) != `{"type":"text","text":"hello"}` {

t.Errorf("unexpected text-block JSON: %s", buf)

}

100

}

package main

import "encoding/base64"

import "encoding/json"

import "flag"

import "io/ioutil"

import "log"

import "net/http"

import "os"

import "path"

import "sort"

import "strings"

import "oscarkilo.com/klex-git/api"

import "oscarkilo.com/klex-git/config"

var dir = flag.String("dir", ".", "Directory to scan and write to.")

var model = flag.String("model", "Gemini 3 Pro", "")

var format = flag.String("format", "text", "text|json|jsonindent")

var dry_run = flag.Bool("dry_run", false, "")

var debug = flag.Bool("debug", false, "")

type Case struct {

Name string

Before []string

After string

Request *api.MessagesRequest

}

func scanForCases() []Case {

entries, err := os.ReadDir(*dir)

if err != nil {

log.Fatalf("Failed to read dir %s: %v", *dir, err)

}

before := make(map[string][]string)

after := make(map[string]string)

for _, entry := range entries {

if entry.IsDir() {

continue

}

if entry.Name() == "system_prompt.txt" {

continue

}

chunks := strings.Split(entry.Name(), ".")

if len(chunks) < 2 {

continue

}

name := strings.Join(chunks[0:len(chunks)-1], ".")

suffix := chunks[len(chunks)-1]

switch suffix {

case "txt", "json", "jpg", "jpeg", "png", "webp":

before[name] = append(before[name], suffix)

case "out":

after[name] = suffix

}

var cases []Case

for name, b := range before {

sort.Slice(b, func(i, j int) bool {

if b[i] == "txt" && b[j] != "txt" {

return true

}

if b[i] != "txt" && b[j] == "txt" {

return false

}

return b[i] < b[j]

})

cases = append(cases, Case{

Name: name,

Before: b,

After: after[name],

})

}

sort.Slice(cases, func(i, j int) bool {

a, b := cases[i], cases[j]

if a.After != "" && b.After == "" {

return true

}

if a.After == "" && b.After != "" {

return false

}

return a.Name < b.Name

})

num_examples := 0

for ; num_examples < len(cases); num_examples++ {

if cases[num_examples].After == "" {

break

}

log.Printf(

"%s:\n num_examples = %d\n num_inputs = %d",

*dir,

num_examples,

len(cases) - num_examples,

)

100

101

return cases

101

return cases

102

}

102

}

103

104

func readFile(fname string) []byte {

104

func readFile(fname string) []byte {

105

data, err := ioutil.ReadFile(path.Join(*dir, fname))

105

data, err := ioutil.ReadFile(path.Join(*dir, fname))

106

if err != nil {

106

if err != nil {

107

log.Fatalf("Failed to read file %s: %v", fname, err)

107

log.Fatalf("Failed to read file %s: %v", fname, err)

108

}

108

}

109

return data

109

return data

110

}

110

}

111

112

func writeFile(fname string, data []byte) error {

112

func writeFile(fname string, data []byte) error {

113

return ioutil.WriteFile(path.Join(*dir, fname), data, 0644)

113

return ioutil.WriteFile(path.Join(*dir, fname), data, 0644)

114

}

114

}

115

116

func copyRequest(req api.MessagesRequest) *api.MessagesRequest {

116

func copyRequest(req api.MessagesRequest) *api.MessagesRequest {

117

bytes, err := json.Marshal(req)

117

bytes, err := json.Marshal(req)

118

if err != nil {

118

if err != nil {

119

log.Fatalf("Failed to copy request: %+v", err)

119

log.Fatalf("Failed to copy request: %+v", err)

120

}

120

}

121

req2 := &api.MessagesRequest{}

121

req2 := &api.MessagesRequest{}

122

err = json.Unmarshal(bytes, req2)

122

err = json.Unmarshal(bytes, req2)

123

if err != nil {

123

if err != nil {

124

log.Fatalf("Failed to copy request: %+v", err)

124

log.Fatalf("Failed to copy request: %+v", err)

125

}

125

}

126

return req2

126

return req2

127

}

127

}

128

129

func main() {

129

func main() {

130

flag.Parse()

130

flag.Parse()

131

132

// Find the API keys and configure a Klex client.

132

// Find the API keys and configure a Klex client.

133

config, err := config.ReadConfig()

133

config, err := config.ReadConfig()

134

if err != nil {

134

if err != nil {

135

log.Fatalf("Failed to read config: %v", err)

135

log.Fatalf("Failed to read config: %v", err)

136

}

136

}

137

client := api.NewClient(config.KlexUrl, config.ApiKey)

137

client := api.NewClient(config.KlexUrl, config.ApiKey)

138

if client == nil {

138

if client == nil {

139

log.Fatalf("Failed to create Klex client")

139

log.Fatalf("Failed to create Klex client")

140

}

140

}

141

142

// Create MessagesRequest objects.

142

// Create MessagesRequest objects.

143

req := api.MessagesRequest{

143

req := api.MessagesRequest{

144

Model: *model,

144

Model: *model,

145

System: string(readFile("system_prompt.txt")),

145

System: string(readFile("system_prompt.txt")),

146

}

146

}

147

cases := scanForCases()

147

cases := scanForCases()

148

for i, c := range cases {

148

for i, c := range cases {

149

user := api.ChatMessage{Role: "user"}

149

user := api.ChatMessage{Role: "user"}

150

text := "Case name: " + c.Name + "\n\n"

150

text := "Case name: " + c.Name + "\n\n"

151

for _, suffix := range c.Before {

151

for _, suffix := range c.Before {

152

switch suffix {

152

switch suffix {

153

case "txt":

153

case "txt":

154

text += string(readFile(c.Name + ".txt"))

154

text += string(readFile(c.Name + ".txt"))

155

case "json":

155

case "json":

156

text += "\n\n```json\n"

156

text += "\n\n```json\n"

157

text += string(readFile(c.Name + ".json"))

157

text += string(readFile(c.Name + ".json"))

158

text += "\n```\n"

158

text += "\n```\n"

159

case "jpg", "jpeg", "png", "webp":

159

case "jpg", "jpeg", "png", "webp":

160

bytes := readFile(c.Name + "." + suffix)

160

bytes := readFile(c.Name + "." + suffix)

161

user.Content = append(user.Content, api.ContentBlock{

161

user.Content = append(user.Content, api.ContentBlock{

162

Type: "image",

162

Type: "document",

163

Source: &api.ContentSource{

163

Source: &api.ContentSource{

164

Type: "base64",

164

Type: "base64",

165

MediaType: http.DetectContentType(bytes),

165

MediaType: http.DetectContentType(bytes),

166

Data: base64.StdEncoding.EncodeToString(bytes),

166

Data: base64.StdEncoding.EncodeToString(bytes),

167

168

})

168

})

169

default:

169

default:

170

log.Fatalf("Unsupported suffix %s in case %s", suffix, c.Name)

170

log.Fatalf("Unsupported suffix %s in case %s", suffix, c.Name)

171

}

171

}

172

}

172

}

173

user.Content = append(user.Content, api.ContentBlock{

173

user.Content = append(user.Content, api.ContentBlock{

174

Type: "text",

174

Type: "text",

175

Text: text,

175

Text: text,

176

})

176

})

177

req.Messages = append(req.Messages, user)

177

req.Messages = append(req.Messages, user)

178

if c.After != "" {

178

if c.After != "" {

179

asst := api.ChatMessage{Role: "assistant"}

179

asst := api.ChatMessage{Role: "assistant"}

180

asst.Content = append(asst.Content, api.ContentBlock{

180

asst.Content = append(asst.Content, api.ContentBlock{

181

Type: "text",

181

Type: "text",

182

Text: string(readFile(c.Name + ".out")),

182

Text: string(readFile(c.Name + ".out")),

183

})

183

})

184

req.Messages = append(req.Messages, asst)

184

req.Messages = append(req.Messages, asst)

185

} else {

185

} else {

186

cases[i].Request = copyRequest(req)

186

cases[i].Request = copyRequest(req)

187

req.Messages = req.Messages[:len(req.Messages)-1]

187

req.Messages = req.Messages[:len(req.Messages)-1]

188

}

188

}

189

}

189

}

190

191

if *dry_run {

191

if *dry_run {

192

log.Printf("Dry run; not sending requests.")

192

log.Printf("Dry run; not sending requests.")

193

return

193

return

194

}

194

}

195

196

// Send 'em.

196

// Send 'em.

197

// TODO: parallelize

197

// TODO: parallelize

198

for _, c := range cases {

198

for _, c := range cases {

199

if c.Request == nil {

199

if c.Request == nil {

200

continue

200

continue

201

}

201

}

202

if *debug {

202

if *debug {

203

log.Printf("Case %s: sending request:", c.Name)

203

log.Printf("Case %s: sending request:", c.Name)

204

enc := json.NewEncoder(os.Stderr)

204

enc := json.NewEncoder(os.Stderr)

205

enc.SetIndent("", " ")

205

enc.SetIndent("", " ")

206

enc.Encode(c.Request)

206

enc.Encode(c.Request)

207

}

207

}

208

res, err := client.Messages(*c.Request)

208

res, err := client.Messages(*c.Request)

209

if err != nil {

209

if err != nil {

210

log.Printf("Case %s: request failed: %+v", c.Name, err)

210

log.Printf("Case %s: request failed: %+v", c.Name, err)

211

continue

211

continue

212

}

212

}

213

if len(res.Content) != 1 {

213

if len(res.Content) != 1 {

214

log.Printf("Case %s: empty response", c.Name)

214

log.Printf("Case %s: empty response", c.Name)

215

continue

215

continue

216

}

216

}

217

c0 := res.Content[0]

217

c0 := res.Content[0]

218

if c0.Type != "text" {

218

if c0.Type != "text" {

219

log.Printf("Case %s: Content[0].Type = %s", c.Name, c0.Type)

219

log.Printf("Case %s: Content[0].Type = %s", c.Name, c0.Type)

220

continue

220

continue

221

}

221

}

222

err = writeFile(c.Name + ".out", append([]byte(c0.Text), '\n'))

222

err = writeFile(c.Name + ".out", append([]byte(c0.Text), '\n'))

223

if err != nil {

223

if err != nil {

224

log.Printf("Case %s: failed to write %s.out: %+v", c.Name, c.Name, err)

224

log.Printf("Case %s: failed to write %s.out: %+v", c.Name, c.Name, err)

225

continue

225

continue

226

}

226

}

227

log.Printf("Case %s: wrote %s.out", c.Name, c.Name)

227

log.Printf("Case %s: wrote %s.out", c.Name, c.Name)

228

}

228

}

229

}

229

}

package main

// This binary runs one LLM inference on one input.

import "encoding/base64"

import "flag"

import "fmt"

import "encoding/json"

import "io/ioutil"

import "log"

import "net/http"

import "os"

import "strings"

import "oscarkilo.com/klex-git/api"

import "oscarkilo.com/klex-git/config"

var model = flag.String("model", "", "overrides .Model, if non-empty")

var system = flag.String("system_file", "", "overrides .System, if non-empty")

var prompt = flag.String("prompt_file", "", "appends to .Messages")

var image = flag.String("image_file", "", "attaches an image to the prompt")

var format = flag.String("format", "text", "text|json|jsonindent")

// guessMimeType returns the MIME type inferred from file contents.

func guessMimeType(b []byte) string {

if len(b) > 512 {

b = b[:512]

}

return http.DetectContentType(b)

}

func main() {

flag.Parse()

// Find the API keys and configure a Klex client.

config, err := config.ReadConfig()

if err != nil {

log.Fatalf("Failed to read config: %v", err)

}

client := api.NewClient(config.KlexUrl, config.ApiKey)

if client == nil {

log.Fatalf("Failed to create Klex client")

}

// Parse stdin as a MessagesRequest object, allowing empty input.

sin, err := ioutil.ReadAll(os.Stdin)

if err != nil {

log.Fatalf("Failed to read stdin: %v", err)

}

if len(sin) == 0 {

sin = []byte("{}")

}

var req api.MessagesRequest

err = json.Unmarshal(sin, &req)

if err != nil {

log.Fatalf("Failed to parse a MessagesRequest from stdin: %v", err)

}

// Use flags to override parts of the request.

if *model != "" {

req.Model = *model

}

if *system != "" {

s, err := ioutil.ReadFile(*system)

if err != nil {

log.Fatalf("Failed to read --system_file %s: %v", *system, err)

}

req.System = string(s)

}

if *image != "" && *prompt == "" {

log.Fatalf("--image_file requires a non-empty --prompt_file, too")

}

if *prompt != "" {

msg := api.ChatMessage{Role: "user"}

if *image != "" {

i, err := ioutil.ReadFile(*image)

if err != nil {

log.Fatalf("Failed to read --image_file %s: %v", *image, err)

}

mime_type := guessMimeType(i)

switch mime_type {

case "image/jpeg", "image/png", "image/gif", "image/webp":

default:

log.Fatalf("Unsupported image type: %s", mime_type)

}

msg.Content = append(msg.Content, api.ContentBlock{

Type: "image",

Type: "document",

Source: &api.ContentSource{

Type: "base64",

MediaType: mime_type,

Data: base64.StdEncoding.EncodeToString(i),

})

}

p, err := ioutil.ReadFile(*prompt)

if err != nil {

log.Fatalf("Failed to read --prompt_file %s: %v", *prompt, err)

}

msg.Content = append(msg.Content, api.ContentBlock{

100

Type: "text",

100

Type: "text",

101

Text: string(p),

101

Text: string(p),

102

})

102

})

103

req.Messages = append(req.Messages, msg)

103

req.Messages = append(req.Messages, msg)

104

}

104

}

105

106

// Get LLM output from Klex.

106

// Get LLM output from Klex.

107

res, err := client.Messages(req)

107

res, err := client.Messages(req)

108

if err != nil {

108

if err != nil {

109

log.Fatalf("Klex f() failure: %v", err)

109

log.Fatalf("Klex f() failure: %v", err)

110

}

110

}

111

112

// Print according to the --format flag.

112

// Print according to the --format flag.

113

out, err := formatResponse(res)

113

out, err := formatResponse(res)

114

if err != nil {

114

if err != nil {

115

log.Fatalf("Failed to format response: %v", err)

115

log.Fatalf("Failed to format response: %v", err)

116

}

116

}

117

fmt.Print(out)

117

fmt.Print(out)

118

}

118

}

119

120

func formatResponse(res *api.MessagesResponse) (string, error) {

120

func formatResponse(res *api.MessagesResponse) (string, error) {

121

switch *format {

121

switch *format {

122

case "text":

122

case "text":

123

var content []string

123

var content []string

124

for _, c := range res.Content {

124

for _, c := range res.Content {

125

if c.Type == "text" {

125

if c.Type == "text" {

126

content = append(content, c.Text + "\n")

126

content = append(content, c.Text + "\n")

127

}

127

}

128

}

128

}

129

return strings.Join(content, "\n"), nil

129

return strings.Join(content, "\n"), nil

130

case "json":

130

case "json":

131

buf, err := json.Marshal(res)

131

buf, err := json.Marshal(res)

132

return string(buf), err

132

return string(buf), err

133

case "jsonindent":

133

case "jsonindent":

134

buf, err := json.MarshalIndent(res, "", " ")

134

buf, err := json.MarshalIndent(res, "", " ")

135

return string(buf), err

135

return string(buf), err

136

default:

136

default:

137

return "", fmt.Errorf("Unsupported --format=%s", *format)

137

return "", fmt.Errorf("Unsupported --format=%s", *format)

138

}

138

}

139

}

139

}