From 10c77854a9ab9b1a379e9f6c51d462175904c896 Mon Sep 17 00:00:00 2001 From: Moby von Briesen Date: Tue, 25 Apr 2023 20:32:08 -0400 Subject: [PATCH] Create "local git client" and "openai client" OpenAIClient can be used to interact with OpenAI's API. Untested at the moment (except with an API key that does not yet have perms) Local git client splits the "git" functionality (e.g. make commits, checkout branches, etc...) away from the "github" client I also removed a lot of duplicate code for the different commands in cmd And created a basic "local issue" command to send a "code change request" to the llm, and receive a code change response. Eventually, I want this to make the corresponding local git changes, for the user to check. TODO: github client should only be responsible for github-specific actions, e.g. opening a PR or listing issues/comments --- cmd/list-comments.go | 21 +---- cmd/list-issues.go | 22 +---- cmd/local-issue.go | 43 ++++++++++ cmd/root.go | 58 ++++++++------ go.mod | 1 + go.sum | 2 + llm/openai.go | 43 ++++++++++ pullpal/common.go | 55 ++++++++++++- vc/git.go | 186 +++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 363 insertions(+), 68 deletions(-) create mode 100644 cmd/local-issue.go create mode 100644 llm/openai.go create mode 100644 vc/git.go diff --git a/cmd/list-comments.go b/cmd/list-comments.go index cfd5f0e..8e5abde 100644 --- a/cmd/list-comments.go +++ b/cmd/list-comments.go @@ -3,11 +3,7 @@ package cmd import ( "fmt" - "github.com/mobyvb/pull-pal/pullpal" - "github.com/mobyvb/pull-pal/vc" - "github.com/spf13/cobra" - "go.uber.org/zap" ) var listCommentsCmd = &cobra.Command{ @@ -18,22 +14,7 @@ var listCommentsCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { cfg := getConfig() - log := zap.L() - - author := vc.Author{ - Email: cfg.selfEmail, - Handle: cfg.selfHandle, - Token: cfg.githubToken, - } - repo := vc.Repository{ - LocalPath: cfg.localRepoPath, - HostDomain: cfg.repoDomain, - Name: cfg.repoName, - Owner: vc.Author{ - Handle: cfg.repoHandle, - }, - } - p, err := pullpal.NewPullPal(cmd.Context(), log.Named("pullpal"), author, repo) + p, err := getPullPal(cmd.Context(), cfg) if err != nil { fmt.Println("error creating new pull pal", err) return diff --git a/cmd/list-issues.go b/cmd/list-issues.go index 47836e3..4e7c330 100644 --- a/cmd/list-issues.go +++ b/cmd/list-issues.go @@ -3,11 +3,7 @@ package cmd import ( "fmt" - "github.com/mobyvb/pull-pal/pullpal" - "github.com/mobyvb/pull-pal/vc" - "github.com/spf13/cobra" - "go.uber.org/zap" ) var listIssuesCmd = &cobra.Command{ @@ -16,24 +12,8 @@ var listIssuesCmd = &cobra.Command{ Long: "Lists github issues meeting the configured criteria", Run: func(cmd *cobra.Command, args []string) { cfg := getConfig() - fmt.Println("list issues called") - log := zap.L() - - author := vc.Author{ - Email: cfg.selfEmail, - Handle: cfg.selfHandle, - Token: cfg.githubToken, - } - repo := vc.Repository{ - LocalPath: cfg.localRepoPath, - HostDomain: cfg.repoDomain, - Name: cfg.repoName, - Owner: vc.Author{ - Handle: cfg.repoHandle, - }, - } - p, err := pullpal.NewPullPal(cmd.Context(), log.Named("pullpal"), author, repo) + p, err := getPullPal(cmd.Context(), cfg) if err != nil { fmt.Println("error creating new pull pal", err) return diff --git a/cmd/local-issue.go b/cmd/local-issue.go new file mode 100644 index 0000000..e8fe91f --- /dev/null +++ b/cmd/local-issue.go @@ -0,0 +1,43 @@ +package cmd + +import ( + "fmt" + + "github.com/mobyvb/pull-pal/vc" + + "github.com/spf13/cobra" +) + +var localIssueCmd = &cobra.Command{ + Use: "local-issue", + Short: "Processes a locally-defined/provided issue rather than remotely reading one from the Github repo", + // TODO csv filepath as arg? + // Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + cfg := getConfig() + + p, err := getPullPal(cmd.Context(), cfg) + if err != nil { + fmt.Println("error creating new pull pal", err) + return + } + fmt.Println("Successfully initialized pull pal") + + newIssue := vc.Issue{ + Subject: "a few updates", + Body: "Add a quote from Frodo to the README.md and index.html files.\nSwitch main.go to port 7777.\nFiles:index.html,README.md,main.go", + Author: vc.Author{ + Handle: "mobyvb", + }, + } + err = p.MakeLocalChange(newIssue) + if err != nil { + fmt.Println("err making local change", err) + return + } + }, +} + +func init() { + rootCmd.AddCommand(localIssueCmd) +} diff --git a/cmd/root.go b/cmd/root.go index a472981..f75e463 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "errors" "fmt" "os" @@ -8,10 +9,10 @@ import ( "github.com/mobyvb/pull-pal/llm" "github.com/mobyvb/pull-pal/pullpal" "github.com/mobyvb/pull-pal/vc" + "go.uber.org/zap" "github.com/spf13/cobra" "github.com/spf13/viper" - "go.uber.org/zap" ) // todo: some of this config definition/usage can be moved to other packages @@ -20,6 +21,7 @@ type config struct { selfHandle string selfEmail string githubToken string + openAIToken string // remote repo info repoDomain string @@ -42,6 +44,7 @@ func getConfig() config { selfHandle: viper.GetString("handle"), selfEmail: viper.GetString("email"), githubToken: viper.GetString("github-token"), + openAIToken: viper.GetString("open-ai-token"), repoDomain: viper.GetString("repo-domain"), repoHandle: viper.GetString("repo-handle"), @@ -57,6 +60,33 @@ func getConfig() config { } } +func getPullPal(ctx context.Context, cfg config) (*pullpal.PullPal, error) { + /* + log, err := zap.NewProduction() + if err != nil { + panic(err) + } + */ + log := zap.L() + + author := vc.Author{ + Email: cfg.selfEmail, + Handle: cfg.selfHandle, + Token: cfg.githubToken, + } + repo := vc.Repository{ + LocalPath: cfg.localRepoPath, + HostDomain: cfg.repoDomain, + Name: cfg.repoName, + Owner: vc.Author{ + Handle: cfg.repoHandle, + }, + } + p, err := pullpal.NewPullPal(ctx, log.Named("pullpal"), author, repo, cfg.openAIToken) + + return p, err +} + // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "pull-pal", @@ -72,29 +102,7 @@ It can be used to: Run: func(cmd *cobra.Command, args []string) { cfg := getConfig() - /* - log, err := zap.NewProduction() - if err != nil { - panic(err) - } - */ - - log := zap.L() - - author := vc.Author{ - Email: cfg.selfEmail, - Handle: cfg.selfHandle, - Token: cfg.githubToken, - } - repo := vc.Repository{ - LocalPath: cfg.localRepoPath, - HostDomain: cfg.repoDomain, - Name: cfg.repoName, - Owner: vc.Author{ - Handle: cfg.repoHandle, - }, - } - p, err := pullpal.NewPullPal(cmd.Context(), log.Named("pullpal"), author, repo) + p, err := getPullPal(cmd.Context(), cfg) if err != nil { fmt.Println("error creating new pull pal", err) return @@ -188,6 +196,7 @@ func init() { rootCmd.PersistentFlags().StringP("handle", "u", "HANDLE", "handle to use for version control actions") rootCmd.PersistentFlags().StringP("email", "e", "EMAIL", "email to use for version control actions") rootCmd.PersistentFlags().StringP("github-token", "t", "GITHUB TOKEN", "token for authenticating Github actions") + rootCmd.PersistentFlags().StringP("open-ai-token", "k", "OPENAI TOKEN", "token for authenticating OpenAI") rootCmd.PersistentFlags().StringP("repo-domain", "d", "github.com", "domain for version control server") rootCmd.PersistentFlags().StringP("repo-handle", "o", "REPO-HANDLE", "handle of repository's owner on version control server") @@ -204,6 +213,7 @@ func init() { viper.BindPFlag("handle", rootCmd.PersistentFlags().Lookup("handle")) viper.BindPFlag("email", rootCmd.PersistentFlags().Lookup("email")) viper.BindPFlag("github-token", rootCmd.PersistentFlags().Lookup("github-token")) + viper.BindPFlag("open-ai-token", rootCmd.PersistentFlags().Lookup("open-ai-token")) viper.BindPFlag("repo-domain", rootCmd.PersistentFlags().Lookup("repo-domain")) viper.BindPFlag("repo-handle", rootCmd.PersistentFlags().Lookup("repo-handle")) diff --git a/go.mod b/go.mod index 9ff6cdb..764e2fc 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/atotto/clipboard v0.1.4 github.com/go-git/go-git/v5 v5.6.1 github.com/google/go-github v17.0.0+incompatible + github.com/sashabaranov/go-openai v1.9.0 github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.15.0 go.uber.org/zap v1.24.0 diff --git a/go.sum b/go.sum index a160420..3ad2548 100644 --- a/go.sum +++ b/go.sum @@ -203,6 +203,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sashabaranov/go-openai v1.9.0 h1:NoiO++IISxxJ1pRc0n7uZvMGMake0G+FJ1XPwXtprsA= +github.com/sashabaranov/go-openai v1.9.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= diff --git a/llm/openai.go b/llm/openai.go new file mode 100644 index 0000000..77a4d1e --- /dev/null +++ b/llm/openai.go @@ -0,0 +1,43 @@ +package llm + +import ( + "context" + "fmt" + + "github.com/sashabaranov/go-openai" +) + +type OpenAIClient struct { + client *openai.Client +} + +func NewOpenAIClient(token string) *OpenAIClient { + return &OpenAIClient{ + client: openai.NewClient(token), + } +} + +func (oc *OpenAIClient) EvaluateCCR(ctx context.Context, req CodeChangeRequest) (res CodeChangeResponse, err error) { + resp, err := oc.client.CreateChatCompletion( + ctx, + openai.ChatCompletionRequest{ + Model: openai.GPT3Dot5Turbo, + Messages: []openai.ChatCompletionMessage{ + { + // TODO is this the correct role for my prompts? + Role: openai.ChatMessageRoleUser, + Content: req.String(), + }, + }, + }, + ) + if err != nil { + fmt.Printf("ChatCompletion error: %v\n", err) + return res, err + } + + // TODO use different choices/different options in different branches/worktrees? + choice := resp.Choices[0].Message.Content + + return ParseCodeChangeResponse(choice), nil +} diff --git a/pullpal/common.go b/pullpal/common.go index db8f904..59f5f91 100644 --- a/pullpal/common.go +++ b/pullpal/common.go @@ -3,6 +3,7 @@ package pullpal import ( "context" "errors" + "fmt" "io/ioutil" "strings" @@ -22,21 +23,29 @@ type PullPal struct { ctx context.Context log *zap.Logger - vcClient vc.VCClient + vcClient vc.VCClient + localGitClient *vc.LocalGitClient + openAIClient *llm.OpenAIClient } // NewPullPal creates a new "pull pal service", including setting up local version control and LLM integrations. -func NewPullPal(ctx context.Context, log *zap.Logger, self vc.Author, repo vc.Repository) (*PullPal, error) { +func NewPullPal(ctx context.Context, log *zap.Logger, self vc.Author, repo vc.Repository, openAIToken string) (*PullPal, error) { ghClient, err := vc.NewGithubClient(ctx, log, self, repo) if err != nil { return nil, err } + localGitClient, err := vc.NewLocalGitClient(self, repo) + if err != nil { + return nil, err + } return &PullPal{ ctx: ctx, log: log, - vcClient: ghClient, + vcClient: ghClient, + localGitClient: localGitClient, + openAIClient: llm.NewOpenAIClient(openAIToken), }, nil } @@ -180,3 +189,43 @@ func (p *PullPal) ListComments(changeID string, handles []string) ([]vc.Comment, return comments, nil } + +func (p *PullPal) MakeLocalChange(issue vc.Issue) error { + // remove file list from issue body + // TODO do this better + parts := strings.Split(issue.Body, "Files:") + issue.Body = parts[0] + + fileList := []string{} + if len(parts) > 1 { + fileList = strings.Split(parts[1], ",") + } + + // get file contents from local git repository + files := []llm.File{} + for _, path := range fileList { + path = strings.TrimSpace(path) + nextFile, err := p.vcClient.GetLocalFile(path) + if err != nil { + return err + } + files = append(files, nextFile) + } + + changeRequest := llm.CodeChangeRequest{ + Subject: issue.Subject, + Body: issue.Body, + IssueID: issue.ID, + Files: files, + } + + res, err := p.openAIClient.EvaluateCCR(p.ctx, changeRequest) + if err != nil { + return err + } + + fmt.Println("response from openai") + fmt.Println(res) + + return nil +} diff --git a/vc/git.go b/vc/git.go new file mode 100644 index 0000000..2e8a521 --- /dev/null +++ b/vc/git.go @@ -0,0 +1,186 @@ +package vc + +import ( + "errors" + "fmt" + "go/format" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/mobyvb/pull-pal/llm" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport/http" +) + +// LocalGitClient represents a service that interacts with a local git repository. +type LocalGitClient struct { + self Author + repo Repository + + worktree *git.Worktree +} + +// NewLocalGitClient initializes a local git client by checking out a repository locally. +func NewLocalGitClient( /*ctx context.Context, log *zap.Logger, */ self Author, repo Repository) (*LocalGitClient, error) { + // clone provided repository to local path + if repo.LocalPath == "" { + return nil, errors.New("local path to clone repository not provided") + } + + // remove local repo if it exists already + err := os.RemoveAll(repo.LocalPath) + if err != nil { + return nil, err + } + + localRepo, err := git.PlainClone(repo.LocalPath, false, &git.CloneOptions{ + URL: repo.SSH(), + // URL: repo.HTTPS(), + Auth: &http.BasicAuth{ + Username: self.Handle, + Password: self.Token, + }, + }) + if err != nil { + return nil, err + } + repo.localRepo = localRepo + + return &LocalGitClient{ + self: self, + repo: repo, + }, nil +} + +func (gc *LocalGitClient) SwitchBranch(branchName string) (err error) { + branchRefName := plumbing.NewBranchReferenceName(branchName) + remoteName := "origin" + + err = gc.repo.localRepo.CreateBranch(&config.Branch{ + Name: branchName, + Remote: remoteName, + Merge: branchRefName, + }) + if err != nil { + return err + } + + return nil +} + +func (gc *LocalGitClient) PushBranch(branchName string) (err error) { + branchRefName := plumbing.NewBranchReferenceName(branchName) + remoteName := "origin" + + // Push the new branch to the remote repository + remote, err := gc.repo.localRepo.Remote(remoteName) + if err != nil { + return err + } + + err = remote.Push(&git.PushOptions{ + RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("%s:refs/heads/%s", branchRefName, branchName))}, + Auth: &http.BasicAuth{ + Username: gc.self.Handle, + Password: gc.self.Token, + }, + }) + if err != nil { + return err + } + + return nil +} + +func (gc *LocalGitClient) GetLocalFile(path string) (llm.File, error) { + fullPath := filepath.Join(gc.repo.LocalPath, path) + + data, err := ioutil.ReadFile(fullPath) + if err != nil { + // if file doesn't exist, just return an empty file + // this means we want to prompt the llm to populate it for the first time + if errors.Is(err, os.ErrNotExist) { + return llm.File{ + Path: path, + Contents: "", + }, nil + } + return llm.File{}, err + } + + return llm.File{ + Path: path, + Contents: string(data), + }, nil +} + +func (gc *LocalGitClient) StartCommit() error { + if gc.worktree != nil { + return errors.New("worktree is not nil - cannot start a new commit") + } + + worktree, err := gc.repo.localRepo.Worktree() + if err != nil { + return err + } + + gc.worktree = worktree + + return nil +} + +// ReplaceOrAddLocalFile updates or adds a file in the locally cloned repo, and applies these changes to the current git worktree. +func (gc *LocalGitClient) ReplaceOrAddLocalFile(newFile llm.File) error { + if gc.worktree == nil { + return errors.New("worktree is nil - StartCommit must be called") + } + + // TODO format non-go files as well + if strings.HasSuffix(newFile.Path, ".go") { + newContents, err := format.Source([]byte(newFile.Contents)) + if err != nil { + return err + } + newFile.Contents = string(newContents) + } + + fullPath := filepath.Join(gc.repo.LocalPath, newFile.Path) + + err := ioutil.WriteFile(fullPath, []byte(newFile.Contents), 0644) + if err != nil { + return err + } + + _, err = gc.worktree.Add(newFile.Path) + + return err +} + +// FinishCommit completes a commit, after which a code change request can be opened or updated. +func (gc *LocalGitClient) FinishCommit(message string) error { + if gc.worktree == nil { + return errors.New("worktree is nil - StartCommit must be called") + } + _, err := gc.worktree.Commit(message, &git.CommitOptions{ + Author: &object.Signature{ + Name: gc.self.Handle, + Email: gc.self.Email, + When: time.Now(), + }, + }) + if err != nil { + return err + } + + // set worktree to nil so a new commit can be started + gc.worktree = nil + + return nil +}