Skip to content

Commit

Permalink
fix: issue when GitHub organization contains more than 30 repos (keda…
Browse files Browse the repository at this point in the history 
…core#5746)

Signed-off-by: Simon Kobler <[email protected]>
Signed-off-by: Simon Kobler <[email protected]>
Co-authored-by: Jorge Turrado Ferrero <[email protected]>
Co-authored-by: Simon Kobler <[email protected]>
Signed-off-by: Jorge Turrado <[email protected]>
  • Loading branch information
@KoblerS @JorTurFer
3 people committed Jul 30, 2024
1 parent c6e6cd2 commit 8386a88
Show file tree
Hide file tree
Showing 2 changed files with 109 additions and 39 deletions.
59 changes: 36 additions & 23 deletions pkg/scalers/github_runner_scaler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -470,31 +470,44 @@ func (s *githubRunnerScaler) getRepositories(ctx context.Context) ([]string, err
return s.metadata.repos, nil
}

var url string
switch s.metadata.runnerScope {
case ORG:
url = fmt.Sprintf("%s/orgs/%s/repos", s.metadata.githubAPIURL, s.metadata.owner)
case REPO:
url = fmt.Sprintf("%s/users/%s/repos", s.metadata.githubAPIURL, s.metadata.owner)
case ENT:
url = fmt.Sprintf("%s/orgs/%s/repos", s.metadata.githubAPIURL, s.metadata.owner)
default:
return nil, fmt.Errorf("runnerScope %s not supported", s.metadata.runnerScope)
}
body, _, err := getGithubRequest(ctx, url, s.metadata, s.httpClient)
if err != nil {
return nil, err
}
page := 1
var repoList []string

var repos []Repo
err = json.Unmarshal(body, &repos)
if err != nil {
return nil, err
}
for {
var url string
switch s.metadata.runnerScope {
case ORG:
url = fmt.Sprintf("%s/orgs/%s/repos?page=%s", s.metadata.githubAPIURL, s.metadata.owner, strconv.Itoa(page))
case REPO:
url = fmt.Sprintf("%s/users/%s/repos?page=%s", s.metadata.githubAPIURL, s.metadata.owner, strconv.Itoa(page))
case ENT:
url = fmt.Sprintf("%s/orgs/%s/repos?page=%s", s.metadata.githubAPIURL, s.metadata.owner, strconv.Itoa(page))
default:
return nil, fmt.Errorf("runnerScope %s not supported", s.metadata.runnerScope)
}

var repoList []string
for _, repo := range repos {
repoList = append(repoList, repo.Name)
body, _, err := getGithubRequest(ctx, url, s.metadata, s.httpClient)
if err != nil {
return nil, err
}

var repos []Repo

err = json.Unmarshal(body, &repos)
if err != nil {
return nil, err
}

for _, repo := range repos {
repoList = append(repoList, repo.Name)
}

// GitHub returned less than 30 repos per page, so consider no repos left
if len(repos) < 30 {
break
}

page++
}

return repoList, nil
Expand Down
89 changes: 73 additions & 16 deletions pkg/scalers/github_runner_scaler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ package scalers

import (
"context"
"crypto/rand"
"encoding/json"
"fmt"
"html/template"
"math/big"
"net/http"
"net/http/httptest"
"strings"
Expand Down Expand Up @@ -155,7 +159,22 @@ func buildQueueJSON() []byte {
return []byte(output)
}

func apiStubHandler(hasRateLeft bool) *httptest.Server {
func generateResponseExceed30Repos() []byte {
var repos []Repo

for i := 0; i < 30; i++ {
var repository Repo
id, _ := rand.Int(rand.Reader, big.NewInt(100000))
repository.ID = int(id.Int64())
repository.Name = "BadRepo"
repos = append(repos, repository)
}

result, _ := json.Marshal(repos)
return result
}

func apiStubHandler(hasRateLeft bool, exceeds30Repos bool) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
futureReset := time.Now()
futureReset = futureReset.Add(time.Minute * 30)
Expand All @@ -178,9 +197,24 @@ func apiStubHandler(hasRateLeft bool) *httptest.Server {
w.WriteHeader(http.StatusOK)
}
}
if strings.HasSuffix(r.URL.String(), "repos") {
_, _ = w.Write([]byte(testGhUserReposResponse))
w.WriteHeader(http.StatusOK)
if strings.Contains(r.URL.String(), "repos?page") {
if exceeds30Repos && strings.HasSuffix(r.URL.String(), "?page=1") {
repos := generateResponseExceed30Repos()
tmpl, err := template.New("repos").Parse(string(repos))
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
err = tmpl.Execute(w, nil)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
} else {
_, _ = w.Write([]byte(testGhUserReposResponse))
w.WriteHeader(http.StatusOK)
}
}
}))
}
Expand All @@ -193,7 +227,7 @@ func apiStubHandler404() *httptest.Server {
}

func TestNewGitHubRunnerScaler_QueueLength_NoRateLeft(t *testing.T) {
var apiStub = apiStubHandler(false)
var apiStub = apiStubHandler(false, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand All @@ -217,7 +251,7 @@ func TestNewGitHubRunnerScaler_QueueLength_NoRateLeft(t *testing.T) {
}

func TestNewGitHubRunnerScaler_QueueLength_SingleRepo(t *testing.T) {
var apiStub = apiStubHandler(true)
var apiStub = apiStubHandler(true, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand All @@ -241,7 +275,7 @@ func TestNewGitHubRunnerScaler_QueueLength_SingleRepo(t *testing.T) {
}

func TestNewGitHubRunnerScaler_QueueLength_SingleRepo_ExtraRunnerLabels(t *testing.T) {
var apiStub = apiStubHandler(true)
var apiStub = apiStubHandler(true, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand All @@ -265,7 +299,7 @@ func TestNewGitHubRunnerScaler_QueueLength_SingleRepo_ExtraRunnerLabels(t *testi
}

func TestNewGitHubRunnerScaler_QueueLength_SingleRepo_LessRunnerLabels(t *testing.T) {
var apiStub = apiStubHandler(true)
var apiStub = apiStubHandler(true, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand Down Expand Up @@ -358,7 +392,7 @@ func TestNewGitHubRunnerScaler_BadURL(t *testing.T) {
}

func TestNewGitHubRunnerScaler_QueueLength_NoRunnerLabels(t *testing.T) {
var apiStub = apiStubHandler(true)
var apiStub = apiStubHandler(true, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand All @@ -382,7 +416,7 @@ func TestNewGitHubRunnerScaler_QueueLength_NoRunnerLabels(t *testing.T) {
}

func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned(t *testing.T) {
var apiStub = apiStubHandler(true)
var apiStub = apiStubHandler(true, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand Down Expand Up @@ -410,7 +444,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned(t *testing.T) {
}

func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned_OneBad(t *testing.T) {
var apiStub = apiStubHandler(true)
var apiStub = apiStubHandler(true, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand Down Expand Up @@ -438,7 +472,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned_OneBad(t *testing.
}

func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledUserRepos(t *testing.T) {
var apiStub = apiStubHandler(true)
var apiStub = apiStubHandler(true, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand All @@ -461,8 +495,31 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledUserRepos(t *testing.
}
}

func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledUserRepos_Exceeds30Entries(t *testing.T) {
var apiStub = apiStubHandler(true, true)

meta := getGitHubTestMetaData(apiStub.URL)

mockGitHubRunnerScaler := githubRunnerScaler{
metadata: meta,
httpClient: http.DefaultClient,
}

mockGitHubRunnerScaler.metadata.labels = []string{"foo", "bar"}

queueLen, err := mockGitHubRunnerScaler.GetWorkflowQueueLength(context.TODO())
if err != nil {
fmt.Println(err)
t.Fail()
}

if queueLen != 2 {
t.Fail()
}
}

func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledOrgRepos(t *testing.T) {
var apiStub = apiStubHandler(true)
var apiStub = apiStubHandler(true, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand All @@ -487,7 +544,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledOrgRepos(t *testing.T
}

func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledEntRepos(t *testing.T) {
var apiStub = apiStubHandler(true)
var apiStub = apiStubHandler(true, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand All @@ -512,7 +569,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledEntRepos(t *testing.T
}

func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledBadRepos(t *testing.T) {
var apiStub = apiStubHandler(true)
var apiStub = apiStubHandler(true, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand All @@ -535,7 +592,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledBadRepos(t *testing.T
}

func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledRepos_NoRate(t *testing.T) {
var apiStub = apiStubHandler(false)
var apiStub = apiStubHandler(false, false)

meta := getGitHubTestMetaData(apiStub.URL)

Expand Down

0 comments on commit 8386a88

Please sign in to comment.