From 8386a8835b59155feb746b6d93dee5d65112e127 Mon Sep 17 00:00:00 2001 From: Simon Kobler <32038731+KoblerS@users.noreply.github.com> Date: Mon, 20 May 2024 12:27:47 +0200 Subject: [PATCH] fix: issue when GitHub organization contains more than 30 repos (#5746) Signed-off-by: Simon Kobler Signed-off-by: Simon Kobler <32038731+KoblerS@users.noreply.github.com> Co-authored-by: Jorge Turrado Ferrero Co-authored-by: Simon Kobler Signed-off-by: Jorge Turrado --- pkg/scalers/github_runner_scaler.go | 59 ++++++++++------ pkg/scalers/github_runner_scaler_test.go | 89 +++++++++++++++++++----- 2 files changed, 109 insertions(+), 39 deletions(-) diff --git a/pkg/scalers/github_runner_scaler.go b/pkg/scalers/github_runner_scaler.go index 9c430341272..d53cd21bd2e 100644 --- a/pkg/scalers/github_runner_scaler.go +++ b/pkg/scalers/github_runner_scaler.go @@ -470,31 +470,44 @@ func (s *githubRunnerScaler) getRepositories(ctx context.Context) ([]string, err return s.metadata.repos, nil } - var url string - switch s.metadata.runnerScope { - case ORG: - url = fmt.Sprintf("%s/orgs/%s/repos", s.metadata.githubAPIURL, s.metadata.owner) - case REPO: - url = fmt.Sprintf("%s/users/%s/repos", s.metadata.githubAPIURL, s.metadata.owner) - case ENT: - url = fmt.Sprintf("%s/orgs/%s/repos", s.metadata.githubAPIURL, s.metadata.owner) - default: - return nil, fmt.Errorf("runnerScope %s not supported", s.metadata.runnerScope) - } - body, _, err := getGithubRequest(ctx, url, s.metadata, s.httpClient) - if err != nil { - return nil, err - } + page := 1 + var repoList []string - var repos []Repo - err = json.Unmarshal(body, &repos) - if err != nil { - return nil, err - } + for { + var url string + switch s.metadata.runnerScope { + case ORG: + url = fmt.Sprintf("%s/orgs/%s/repos?page=%s", s.metadata.githubAPIURL, s.metadata.owner, strconv.Itoa(page)) + case REPO: + url = fmt.Sprintf("%s/users/%s/repos?page=%s", s.metadata.githubAPIURL, s.metadata.owner, strconv.Itoa(page)) + case ENT: + url = fmt.Sprintf("%s/orgs/%s/repos?page=%s", s.metadata.githubAPIURL, s.metadata.owner, strconv.Itoa(page)) + default: + return nil, fmt.Errorf("runnerScope %s not supported", s.metadata.runnerScope) + } - var repoList []string - for _, repo := range repos { - repoList = append(repoList, repo.Name) + body, _, err := getGithubRequest(ctx, url, s.metadata, s.httpClient) + if err != nil { + return nil, err + } + + var repos []Repo + + err = json.Unmarshal(body, &repos) + if err != nil { + return nil, err + } + + for _, repo := range repos { + repoList = append(repoList, repo.Name) + } + + // GitHub returned less than 30 repos per page, so consider no repos left + if len(repos) < 30 { + break + } + + page++ } return repoList, nil diff --git a/pkg/scalers/github_runner_scaler_test.go b/pkg/scalers/github_runner_scaler_test.go index 2d03e81ecb1..fc1babdddc2 100644 --- a/pkg/scalers/github_runner_scaler_test.go +++ b/pkg/scalers/github_runner_scaler_test.go @@ -2,7 +2,11 @@ package scalers import ( "context" + "crypto/rand" + "encoding/json" "fmt" + "html/template" + "math/big" "net/http" "net/http/httptest" "strings" @@ -155,7 +159,22 @@ func buildQueueJSON() []byte { return []byte(output) } -func apiStubHandler(hasRateLeft bool) *httptest.Server { +func generateResponseExceed30Repos() []byte { + var repos []Repo + + for i := 0; i < 30; i++ { + var repository Repo + id, _ := rand.Int(rand.Reader, big.NewInt(100000)) + repository.ID = int(id.Int64()) + repository.Name = "BadRepo" + repos = append(repos, repository) + } + + result, _ := json.Marshal(repos) + return result +} + +func apiStubHandler(hasRateLeft bool, exceeds30Repos bool) *httptest.Server { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { futureReset := time.Now() futureReset = futureReset.Add(time.Minute * 30) @@ -178,9 +197,24 @@ func apiStubHandler(hasRateLeft bool) *httptest.Server { w.WriteHeader(http.StatusOK) } } - if strings.HasSuffix(r.URL.String(), "repos") { - _, _ = w.Write([]byte(testGhUserReposResponse)) - w.WriteHeader(http.StatusOK) + if strings.Contains(r.URL.String(), "repos?page") { + if exceeds30Repos && strings.HasSuffix(r.URL.String(), "?page=1") { + repos := generateResponseExceed30Repos() + tmpl, err := template.New("repos").Parse(string(repos)) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + err = tmpl.Execute(w, nil) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + } else { + _, _ = w.Write([]byte(testGhUserReposResponse)) + w.WriteHeader(http.StatusOK) + } } })) } @@ -193,7 +227,7 @@ func apiStubHandler404() *httptest.Server { } func TestNewGitHubRunnerScaler_QueueLength_NoRateLeft(t *testing.T) { - var apiStub = apiStubHandler(false) + var apiStub = apiStubHandler(false, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -217,7 +251,7 @@ func TestNewGitHubRunnerScaler_QueueLength_NoRateLeft(t *testing.T) { } func TestNewGitHubRunnerScaler_QueueLength_SingleRepo(t *testing.T) { - var apiStub = apiStubHandler(true) + var apiStub = apiStubHandler(true, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -241,7 +275,7 @@ func TestNewGitHubRunnerScaler_QueueLength_SingleRepo(t *testing.T) { } func TestNewGitHubRunnerScaler_QueueLength_SingleRepo_ExtraRunnerLabels(t *testing.T) { - var apiStub = apiStubHandler(true) + var apiStub = apiStubHandler(true, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -265,7 +299,7 @@ func TestNewGitHubRunnerScaler_QueueLength_SingleRepo_ExtraRunnerLabels(t *testi } func TestNewGitHubRunnerScaler_QueueLength_SingleRepo_LessRunnerLabels(t *testing.T) { - var apiStub = apiStubHandler(true) + var apiStub = apiStubHandler(true, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -358,7 +392,7 @@ func TestNewGitHubRunnerScaler_BadURL(t *testing.T) { } func TestNewGitHubRunnerScaler_QueueLength_NoRunnerLabels(t *testing.T) { - var apiStub = apiStubHandler(true) + var apiStub = apiStubHandler(true, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -382,7 +416,7 @@ func TestNewGitHubRunnerScaler_QueueLength_NoRunnerLabels(t *testing.T) { } func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned(t *testing.T) { - var apiStub = apiStubHandler(true) + var apiStub = apiStubHandler(true, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -410,7 +444,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned(t *testing.T) { } func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned_OneBad(t *testing.T) { - var apiStub = apiStubHandler(true) + var apiStub = apiStubHandler(true, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -438,7 +472,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned_OneBad(t *testing. } func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledUserRepos(t *testing.T) { - var apiStub = apiStubHandler(true) + var apiStub = apiStubHandler(true, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -461,8 +495,31 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledUserRepos(t *testing. } } +func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledUserRepos_Exceeds30Entries(t *testing.T) { + var apiStub = apiStubHandler(true, true) + + meta := getGitHubTestMetaData(apiStub.URL) + + mockGitHubRunnerScaler := githubRunnerScaler{ + metadata: meta, + httpClient: http.DefaultClient, + } + + mockGitHubRunnerScaler.metadata.labels = []string{"foo", "bar"} + + queueLen, err := mockGitHubRunnerScaler.GetWorkflowQueueLength(context.TODO()) + if err != nil { + fmt.Println(err) + t.Fail() + } + + if queueLen != 2 { + t.Fail() + } +} + func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledOrgRepos(t *testing.T) { - var apiStub = apiStubHandler(true) + var apiStub = apiStubHandler(true, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -487,7 +544,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledOrgRepos(t *testing.T } func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledEntRepos(t *testing.T) { - var apiStub = apiStubHandler(true) + var apiStub = apiStubHandler(true, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -512,7 +569,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledEntRepos(t *testing.T } func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledBadRepos(t *testing.T) { - var apiStub = apiStubHandler(true) + var apiStub = apiStubHandler(true, false) meta := getGitHubTestMetaData(apiStub.URL) @@ -535,7 +592,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledBadRepos(t *testing.T } func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledRepos_NoRate(t *testing.T) { - var apiStub = apiStubHandler(false) + var apiStub = apiStubHandler(false, false) meta := getGitHubTestMetaData(apiStub.URL)