fix: issue when GitHub organization contains more than 30 repos

CHANGELOG.md

@@ -70,6 +70,7 @@ Here is an overview of all new **experimental** features:
 ### Improvements
 
 - **GCP Scalers**: Added custom time horizon in GCP scalers ([#5778](https://github.com/kedacore/keda/issues/5778))
+- **GitHub Scaler**: Fixed pagination, fetching repository list ([#5738](https://github.com/kedacore/keda/issues/5738))
 - **Kafka**: Fix logic to scale to zero on invalid offset even with earliest offsetResetPolicy ([#5689](https://github.com/kedacore/keda/issues/5689))
 
 ### Fixes

pkg/scalers/github_runner_scaler.go

@@ -470,31 +470,44 @@ func (s *githubRunnerScaler) getRepositories(ctx context.Context) ([]string, err
 		return s.metadata.repos, nil
 	}
 
-	var url string
-	switch s.metadata.runnerScope {
-	case ORG:
-		url = fmt.Sprintf("%s/orgs/%s/repos", s.metadata.githubAPIURL, s.metadata.owner)
-	case REPO:
-		url = fmt.Sprintf("%s/users/%s/repos", s.metadata.githubAPIURL, s.metadata.owner)
-	case ENT:
-		url = fmt.Sprintf("%s/orgs/%s/repos", s.metadata.githubAPIURL, s.metadata.owner)
-	default:
-		return nil, fmt.Errorf("runnerScope %s not supported", s.metadata.runnerScope)
-	}
-	body, _, err := getGithubRequest(ctx, url, s.metadata, s.httpClient)
-	if err != nil {
-		return nil, err
-	}
+	page := 1
+	var repoList []string
 
-	var repos []Repo
-	err = json.Unmarshal(body, &repos)
-	if err != nil {
-		return nil, err
-	}
+	for {
+		var url string
+		switch s.metadata.runnerScope {
+		case ORG:
+			url = fmt.Sprintf("%s/orgs/%s/repos?page=%s", s.metadata.githubAPIURL, s.metadata.owner, strconv.Itoa(page))
+		case REPO:
+			url = fmt.Sprintf("%s/users/%s/repos?page=%s", s.metadata.githubAPIURL, s.metadata.owner, strconv.Itoa(page))
+		case ENT:
+			url = fmt.Sprintf("%s/orgs/%s/repos?page=%s", s.metadata.githubAPIURL, s.metadata.owner, strconv.Itoa(page))
+		default:
+			return nil, fmt.Errorf("runnerScope %s not supported", s.metadata.runnerScope)
+		}
 
-	var repoList []string
-	for _, repo := range repos {
-		repoList = append(repoList, repo.Name)
+		body, _, err := getGithubRequest(ctx, url, s.metadata, s.httpClient)
+		if err != nil {
+			return nil, err
+		}
+
+		var repos []Repo
+
+		err = json.Unmarshal(body, &repos)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, repo := range repos {
+			repoList = append(repoList, repo.Name)
+		}
+
+		// GitHub returned less than 30 repos per page, so consider no repos left
+		if len(repos) < 30 {
+			break
+		}
+
+		page++
 	}
 
 	return repoList, nil

pkg/scalers/github_runner_scaler_test.go

@@ -2,7 +2,11 @@ package scalers
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/json"
 	"fmt"
+	"html/template"
+	"math/big"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -155,7 +159,22 @@ func buildQueueJSON() []byte {
 	return []byte(output)
 }
 
-func apiStubHandler(hasRateLeft bool) *httptest.Server {
+func generateResponseExceed30Repos() []byte {
+	var repos []Repo
+
+	for i := 0; i < 30; i++ {
+		var repository Repo
+		id, _ := rand.Int(rand.Reader, big.NewInt(100000))
+		repository.ID = int(id.Int64())
+		repository.Name = "BadRepo"
+		repos = append(repos, repository)
+	}
+
+	result, _ := json.Marshal(repos)
+	return result
+}
+
+func apiStubHandler(hasRateLeft bool, exceeds30Repos bool) *httptest.Server {
 	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		futureReset := time.Now()
 		futureReset = futureReset.Add(time.Minute * 30)
@@ -178,9 +197,24 @@ func apiStubHandler(hasRateLeft bool) *httptest.Server {
 				w.WriteHeader(http.StatusOK)
 			}
 		}
-		if strings.HasSuffix(r.URL.String(), "repos") {
-			_, _ = w.Write([]byte(testGhUserReposResponse))
-			w.WriteHeader(http.StatusOK)
+		if strings.Contains(r.URL.String(), "repos?page") {
+			if exceeds30Repos && strings.HasSuffix(r.URL.String(), "?page=1") {
+				repos := generateResponseExceed30Repos()
+				tmpl, err := template.New("repos").Parse(string(repos))
+				if err != nil {
+					w.WriteHeader(http.StatusInternalServerError)
+					return
+				}
+				err = tmpl.Execute(w, nil)
+				if err != nil {
+					w.WriteHeader(http.StatusInternalServerError)
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+			} else {
+				_, _ = w.Write([]byte(testGhUserReposResponse))
+				w.WriteHeader(http.StatusOK)
+			}
 		}
 	}))
 }
@@ -193,7 +227,7 @@ func apiStubHandler404() *httptest.Server {
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_NoRateLeft(t *testing.T) {
-	var apiStub = apiStubHandler(false)
+	var apiStub = apiStubHandler(false, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -217,7 +251,7 @@ func TestNewGitHubRunnerScaler_QueueLength_NoRateLeft(t *testing.T) {
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_SingleRepo(t *testing.T) {
-	var apiStub = apiStubHandler(true)
+	var apiStub = apiStubHandler(true, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -241,7 +275,7 @@ func TestNewGitHubRunnerScaler_QueueLength_SingleRepo(t *testing.T) {
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_SingleRepo_ExtraRunnerLabels(t *testing.T) {
-	var apiStub = apiStubHandler(true)
+	var apiStub = apiStubHandler(true, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -265,7 +299,7 @@ func TestNewGitHubRunnerScaler_QueueLength_SingleRepo_ExtraRunnerLabels(t *testi
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_SingleRepo_LessRunnerLabels(t *testing.T) {
-	var apiStub = apiStubHandler(true)
+	var apiStub = apiStubHandler(true, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -358,7 +392,7 @@ func TestNewGitHubRunnerScaler_BadURL(t *testing.T) {
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_NoRunnerLabels(t *testing.T) {
-	var apiStub = apiStubHandler(true)
+	var apiStub = apiStubHandler(true, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -382,7 +416,7 @@ func TestNewGitHubRunnerScaler_QueueLength_NoRunnerLabels(t *testing.T) {
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned(t *testing.T) {
-	var apiStub = apiStubHandler(true)
+	var apiStub = apiStubHandler(true, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -410,7 +444,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned(t *testing.T) {
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned_OneBad(t *testing.T) {
-	var apiStub = apiStubHandler(true)
+	var apiStub = apiStubHandler(true, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -438,7 +472,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_Assigned_OneBad(t *testing.
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledUserRepos(t *testing.T) {
-	var apiStub = apiStubHandler(true)
+	var apiStub = apiStubHandler(true, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -461,8 +495,31 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledUserRepos(t *testing.
 	}
 }
 
+func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledUserRepos_Exceeds30Entries(t *testing.T) {
+	var apiStub = apiStubHandler(true, true)
+
+	meta := getGitHubTestMetaData(apiStub.URL)
+
+	mockGitHubRunnerScaler := githubRunnerScaler{
+		metadata:   meta,
+		httpClient: http.DefaultClient,
+	}
+
+	mockGitHubRunnerScaler.metadata.labels = []string{"foo", "bar"}
+
+	queueLen, err := mockGitHubRunnerScaler.GetWorkflowQueueLength(context.TODO())
+	if err != nil {
+		fmt.Println(err)
+		t.Fail()
+	}
+
+	if queueLen != 2 {
+		t.Fail()
+	}
+}
+
 func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledOrgRepos(t *testing.T) {
-	var apiStub = apiStubHandler(true)
+	var apiStub = apiStubHandler(true, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -487,7 +544,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledOrgRepos(t *testing.T
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledEntRepos(t *testing.T) {
-	var apiStub = apiStubHandler(true)
+	var apiStub = apiStubHandler(true, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -512,7 +569,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledEntRepos(t *testing.T
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledBadRepos(t *testing.T) {
-	var apiStub = apiStubHandler(true)
+	var apiStub = apiStubHandler(true, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)
 
@@ -535,7 +592,7 @@ func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledBadRepos(t *testing.T
 }
 
 func TestNewGitHubRunnerScaler_QueueLength_MultiRepo_PulledRepos_NoRate(t *testing.T) {
-	var apiStub = apiStubHandler(false)
+	var apiStub = apiStubHandler(false, false)
 
 	meta := getGitHubTestMetaData(apiStub.URL)