use json parsing for prompt responses

llm/common.go

@@ -1,13 +1,9 @@
 package llm
 
-import (
-	"strings"
-)
-
 // File represents a file in a git repository.
 type File struct {
-	Path     string
-	Contents string
+	Path     string `json:"path"`
+	Contents string `json:"contents"`
 }
 
 type ResponseType int
@@ -28,8 +24,8 @@ type CodeChangeRequest struct {
 
 // CodeChangeResponse contains data derived from an LLM response to a prompt generated via a CodeChangeRequest.
 type CodeChangeResponse struct {
-	Files []File
-	Notes string
+	Files []File `json:"files"`
+	Notes string `json:"notes"`
 }
 
 // TODO support threads
@@ -40,42 +36,7 @@ type DiffCommentRequest struct {
 }
 
 type DiffCommentResponse struct {
-	Type   ResponseType
-	Answer string
-	File   File
-}
-
-// parseFiles process the "files" subsection of the LLM's response. It is a helper for GetCodeChangeResponse.
-func parseFiles(filesSection string) []File {
-	fileStringList := strings.Split(filesSection, "ppname:")
-	if len(fileStringList) < 2 {
-		return []File{}
-	}
-	// first item in the list is just gonna be "Files:"
-	fileStringList = fileStringList[1:]
-
-	replacer := strings.NewReplacer(
-		"\\n", "\n",
-		"\\\"", "\"",
-		"```", "",
-	)
-	fileList := make([]File, len(fileStringList))
-	for i, f := range fileStringList {
-		fileParts := strings.Split(f, "ppcontents:")
-		if len(fileParts) < 2 {
-			continue
-		}
-		path := replacer.Replace(fileParts[0])
-		path = strings.TrimSpace(path)
-
-		contents := replacer.Replace(fileParts[1])
-		contents = strings.TrimSpace(contents)
-
-		fileList[i] = File{
-			Path:     path,
-			Contents: contents,
-		}
-	}
-
-	return fileList
+	Type     ResponseType `json:"responseType"`
+	Response string       `json:"response"`
+	File     File         `json:"file"`
 }

llm/diffcomment.go

@@ -2,7 +2,7 @@ package llm
 
 import (
 	"bytes"
-	"strings"
+	"encoding/json"
 	"text/template"
 )
 
@@ -40,14 +40,14 @@ func (res DiffCommentResponse) String() string {
 	out := ""
 	if res.Type == ResponseAnswer {
 		out += "Type: Answer\n"
-		out += res.Answer
+		out += res.Response
 		return out
 	}
 
 	out += "Type: Code Change\n"
 
 	out += "Response:\n"
-	out += res.Answer + "\n\n"
+	out += res.Response + "\n\n"
 	out += "Files:\n"
 	out += res.File.Path + ":\n```\n"
 	out += res.File.Contents + "\n```\n"
@@ -55,36 +55,8 @@ func (res DiffCommentResponse) String() string {
 	return out
 }
 
-func ParseDiffCommentResponse(llmResponse string) DiffCommentResponse {
-	llmResponse = strings.TrimSpace(llmResponse)
-	if llmResponse[0] == 'A' {
-		answer := strings.TrimSpace(llmResponse[1:])
-		return DiffCommentResponse{
-			Type:   ResponseAnswer,
-			Answer: answer,
-		}
-	}
-	parts := strings.Split(llmResponse, "ppresponse:")
-
-	filesSection := ""
-	if len(parts) > 0 {
-		filesSection = parts[0]
-	}
-
-	answer := ""
-	if len(parts) > 1 {
-		answer = strings.TrimSpace(parts[1])
-	}
-
-	files := parseFiles(filesSection)
-	f := File{}
-	if len(files) > 0 {
-		f = files[0]
-	}
-
-	return DiffCommentResponse{
-		Type:   ResponseCodeChange,
-		Answer: answer,
-		File:   f,
-	}
+func ParseDiffCommentResponse(llmResponse string) (DiffCommentResponse, error) {
+	var response DiffCommentResponse
+	err := json.Unmarshal([]byte(llmResponse), &response)
+	return response, err
 }

llm/issue.go

@@ -2,7 +2,7 @@ package llm
 
 import (
 	"bytes"
-	"strings"
+	"encoding/json"
 	"text/template"
 )
 
@@ -50,22 +50,8 @@ func (res CodeChangeResponse) String() string {
 }
 
 // ParseCodeChangeResponse parses the LLM's response to CodeChangeRequest (string) into a CodeChangeResponse.
-func ParseCodeChangeResponse(llmResponse string) CodeChangeResponse {
-	sections := strings.Split(llmResponse, "ppnotes:")
-
-	filesSection := ""
-	if len(sections) > 0 {
-		filesSection = sections[0]
-	}
-	notes := ""
-	if len(sections) > 1 {
-		notes = strings.TrimSpace(sections[1])
-	}
-
-	files := parseFiles(filesSection)
-
-	return CodeChangeResponse{
-		Files: files,
-		Notes: notes,
-	}
+func ParseCodeChangeResponse(llmResponse string) (CodeChangeResponse, error) {
+	var response CodeChangeResponse
+	err := json.Unmarshal([]byte(llmResponse), &response)
+	return response, err
 }

llm/openai.go

@@ -44,10 +44,9 @@ func (oc *OpenAIClient) EvaluateCCR(ctx context.Context, model string, req CodeC
 
 	choice := resp.Choices[0].Message.Content
 
-	// TODO make debug log when I figure out how to config that
 	oc.log.Info("got response from llm", zap.String("output", choice))
 
-	return ParseCodeChangeResponse(choice), nil
+	return ParseCodeChangeResponse(choice)
 }
 
 func (oc *OpenAIClient) EvaluateDiffComment(ctx context.Context, model string, req DiffCommentRequest) (res DiffCommentResponse, err error) {
@@ -76,5 +75,5 @@ func (oc *OpenAIClient) EvaluateDiffComment(ctx context.Context, model string, r
 	// TODO make debug log when I figure out how to config that
 	oc.log.Info("got response from llm", zap.String("output", choice))
 
-	return ParseDiffCommentResponse(choice), nil
+	return ParseDiffCommentResponse(choice)
 }

llm/prompts/code-change-request.tmpl

@@ -12,13 +12,14 @@ Subject: {{ .Subject }}
 Body:
 {{ .Body }}
 
-Respond in the exact format:
-Files:
+Respond in a parseable JSON format based on the following template:
+```
+{
+"files": [
 {{ range $index, $file := .Files }}
-ppname: {{ $file.Path }}
-ppcontents:
-[new {{ $file.Path }} contents]
+    {"path": "{{ $file.Path }}", "contents": "[new {{ $file.Path }} contents]"},
 {{ end }}
-
-ppnotes:
-[additional context about your changes]
+],
+"notes": "[additional context about your changes]"
+}
+```

llm/prompts/comment-diff-request.tmpl

@@ -16,19 +16,25 @@ Comment:
 The above is information about a comment left on a file. The diff contains information about the precise location of the comment.
 
 First, determine if the comment is a question or a request for changes.
-If the comment is a question, come up with an answer, and respond exactly as outlined directly below "Response Template A", starting with "Q".
-If the comment is a request, modify the file provided at the beginning of the message, and respond exactly as outlined directly below "Response Template B", starting with "R".
+If the comment is a question, come up with an answer, and respond exactly as outlined directly below "Response Template A".
+If the comment is a request, modify the file provided at the beginning of the message, and respond exactly as outlined directly below "Response Template B".
 
-Response Template A:
-Q
-[your answer]
+Response Template A (Respond in a parseable JSON format):
+```
+{
+"responseType": 0,
+"response": "[your answer]"
+}
+```
 
-Response Template B:
-R
-Files:
-ppname: {{ .File.Path }}
-ppcontents:
-[new {{ .File.Path }} contents]
-
-ppresponse:
-[additional context about your changes]
+Response Template B (Respond in a parseable JSON format):
+```
+{
+"responseType": 1,
+"file": {
+    "path": "{{ .File.Path }}",
+    "contents": "[new {{ .File.Path }} contents]"
+},
+"response": "[additional context about your changes]"
+}
+```

pullpal/common.go

@@ -298,7 +298,7 @@ func (p *pullPalRepo) handleComment(comment vc.Comment) error {
 		}
 	}
 
-	err = p.ghClient.RespondToComment(comment.PRNumber, comment.ID, diffCommentResponse.Answer)
+	err = p.ghClient.RespondToComment(comment.PRNumber, comment.ID, diffCommentResponse.Response)
 	if err != nil {
 		p.log.Error("error commenting on issue", zap.Error(err))
 		return err