add debug dir where prompt output is written

cmd/root.go

@@ -33,6 +33,7 @@ type config struct {
 	usersToListenTo     []string
 	requiredIssueLabels []string
 	waitDuration        time.Duration
+	debugDir            string
 }
 
 func getConfig() config {
@@ -49,6 +50,7 @@ func getConfig() config {
 		usersToListenTo:     viper.GetStringSlice("users-to-listen-to"),
 		requiredIssueLabels: viper.GetStringSlice("required-issue-labels"),
 		waitDuration:        viper.GetDuration("wait-duration"),
+		debugDir:            viper.GetString("debug-dir"),
 	}
 }
 
@@ -79,6 +81,7 @@ func getPullPal(ctx context.Context, cfg config) (*pullpal.PullPal, error) {
 		// TODO configurable model
 		Model:       openai.GPT4,
 		OpenAIToken: cfg.openAIToken,
+		DebugDir:    cfg.debugDir,
 	}
 	p, err := pullpal.NewPullPal(ctx, log.Named("pullpal"), ppCfg)
 
@@ -135,6 +138,7 @@ func init() {
 	rootCmd.PersistentFlags().StringSliceP("users-to-listen-to", "a", []string{}, "a list of Github users that Pull Pal will respond to")
 	rootCmd.PersistentFlags().StringSliceP("required-issue-labels", "i", []string{}, "a list of labels that are required for Pull Pal to select an issue")
 	rootCmd.PersistentFlags().Duration("wait-time", 30*time.Second, "the amount of time Pull Pal should wait when no issues or comments are found to address")
+	rootCmd.PersistentFlags().StringP("debug-dir", "d", "", "the path to use for the pull pal debug directory")
 
 	viper.BindPFlag("handle", rootCmd.PersistentFlags().Lookup("handle"))
 	viper.BindPFlag("email", rootCmd.PersistentFlags().Lookup("email"))
@@ -148,6 +152,7 @@ func init() {
 	viper.BindPFlag("users-to-listen-to", rootCmd.PersistentFlags().Lookup("users-to-listen-to"))
 	viper.BindPFlag("required-issue-labels", rootCmd.PersistentFlags().Lookup("required-issue-labels"))
 	viper.BindPFlag("wait-time", rootCmd.PersistentFlags().Lookup("wait-time"))
+	viper.BindPFlag("debug-dir", rootCmd.PersistentFlags().Lookup("debug-dir"))
 }
 
 func initConfig() {

llm/common.go

@@ -33,6 +33,7 @@ type DiffCommentRequest struct {
 	File     File
 	Contents string
 	Diff     string
+	PRNumber int
 }
 
 type DiffCommentResponse struct {

llm/openai.go

@@ -2,6 +2,11 @@ package llm
 
 import (
 	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"time"
 
 	"github.com/sashabaranov/go-openai"
 	"go.uber.org/zap"
@@ -10,14 +15,16 @@ import (
 type OpenAIClient struct {
 	log          *zap.Logger
 	client       *openai.Client
+	debugDir     string
 	defaultModel string
 }
 
-func NewOpenAIClient(log *zap.Logger, defaultModel, token string) *OpenAIClient {
+func NewOpenAIClient(log *zap.Logger, defaultModel, token, debugDir string) *OpenAIClient {
 	return &OpenAIClient{
 		log:          log,
 		client:       openai.NewClient(token),
 		defaultModel: defaultModel,
+		debugDir:     debugDir,
 	}
 }
 
@@ -44,7 +51,22 @@ func (oc *OpenAIClient) EvaluateCCR(ctx context.Context, model string, req CodeC
 
 	choice := resp.Choices[0].Message.Content
 
-	oc.log.Info("got response from llm", zap.String("output", choice))
+	oc.log.Info("got response from llm")
+	if oc.debugDir != "" {
+		subdir := path.Join(oc.debugDir, "codechangeresponse")
+		err = os.MkdirAll(subdir, os.ModePerm)
+		if err != nil {
+			oc.log.Error("failed to ensure debug directory existed", zap.String("filepath", subdir), zap.Error(err))
+		} else {
+			fullPath := path.Join(subdir, fmt.Sprintf("%d-%d.json", req.IssueNumber, time.Now().Unix()))
+			err = ioutil.WriteFile(fullPath, []byte(choice), 0644)
+			if err != nil {
+				oc.log.Error("failed to write response to debug file", zap.String("filepath", fullPath), zap.Error(err))
+			} else {
+				oc.log.Info("response written to debug file", zap.String("filepath", fullPath))
+			}
+		}
+	}
 
 	return ParseCodeChangeResponse(choice)
 }
@@ -72,8 +94,23 @@ func (oc *OpenAIClient) EvaluateDiffComment(ctx context.Context, model string, r
 
 	choice := resp.Choices[0].Message.Content
 
-	// TODO make debug log when I figure out how to config that
 	oc.log.Info("got response from llm", zap.String("output", choice))
+	// TODO
+	if oc.debugDir != "" {
+		subdir := path.Join(oc.debugDir, "diffcommentresponse")
+		err = os.MkdirAll(subdir, os.ModePerm)
+		if err != nil {
+			oc.log.Error("failed to ensure debug directory existed", zap.String("filepath", subdir), zap.Error(err))
+		} else {
+			fullPath := path.Join(subdir, fmt.Sprintf("%d-%d.json", req.PRNumber, time.Now().Unix()))
+			err = ioutil.WriteFile(fullPath, []byte(choice), 0644)
+			if err != nil {
+				oc.log.Error("failed to write response to debug file", zap.String("filepath", fullPath), zap.Error(err))
+			} else {
+				oc.log.Info("response written to debug file", zap.String("filepath", fullPath))
+			}
+		}
+	}
 
 	return ParseDiffCommentResponse(choice)
 }

pullpal/common.go

@@ -26,6 +26,7 @@ type Config struct {
 	ListIssueOptions vc.ListIssueOptions
 	Model            string
 	OpenAIToken      string
+	DebugDir         string
 }
 
 // PullPal is the service responsible for:
@@ -54,7 +55,7 @@ type pullPalRepo struct {
 
 // NewPullPal creates a new "pull pal service", including setting up local version control and LLM integrations.
 func NewPullPal(ctx context.Context, log *zap.Logger, cfg Config) (*PullPal, error) {
-	openAIClient := llm.NewOpenAIClient(log.Named("openaiClient"), cfg.Model, cfg.OpenAIToken)
+	openAIClient := llm.NewOpenAIClient(log.Named("openaiClient"), cfg.Model, cfg.OpenAIToken, cfg.DebugDir)
 
 	ppRepos := []pullPalRepo{}
 	for _, r := range cfg.Repos {
@@ -260,6 +261,7 @@ func (p *pullPalRepo) handleComment(comment vc.Comment) error {
 		File:     file,
 		Contents: comment.Body,
 		Diff:     comment.DiffHunk,
+		PRNumber: comment.PRNumber,
 	}
 	p.log.Info("diff comment request", zap.String("req", diffCommentRequest.String()))