feat/reconcliation of branches (#2498)

* Bring back docs on local development

* Improve local dev docs

* Agent task for moving from callbacks to webhooks

* Move from callback to webhooks, again

* Make repos webhook async; dont comment erorr when installed to all repos

* remove unecessary check

* unnecessary comments

* remove commented out code, ensure code parameter handled gracefully

* fix error

* ensure no unnecessary ai summaries generated

---------

Co-authored-by: Igor Zalutski <izalutski@gmail.com>

Author: motatoes

backend/controllers/github.go

@@ -87,11 +87,15 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 			"added", len(event.RepositoriesAdded),
 			"removed", len(event.RepositoriesRemoved),
 		)
-		if err := handleInstallationRepositoriesEvent(c.Request.Context(), gh, event, appId64); err != nil {
-			slog.Error("Failed to handle installation repositories event", "error", err)
-			c.String(http.StatusAccepted, "Failed to handle webhook event.")
-			return
-		}
+
+		// Run in goroutine to avoid webhook timeouts for large installations
+		go func(ctx context.Context) {
+			defer logging.InheritRequestLogger(ctx)()
+			// Use background context so work continues after HTTP response
+			if err := handleInstallationRepositoriesEvent(context.Background(), gh, event, appId64); err != nil {
+				slog.Error("Failed to handle installation repositories event", "error", err)
+			}
+		}(c.Request.Context())
 	case *github.PushEvent:
 		slog.Info("Processing PushEvent",
 			"repo", *event.Repo.FullName,

backend/controllers/github_callback.go

@@ -31,6 +31,10 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
 	code := ""
 	if codeExists && len(codeParams) > 0 && len(codeParams[0]) > 0 {
 		code = codeParams[0]
+	} else {
+		slog.Debug("No code parameter found, probably a setup update, going to return success since we are relying on webhooks now")
+		c.HTML(http.StatusOK, "github_success.tmpl", gin.H{})
+		return
 	}
 
 	appId := c.Request.URL.Query().Get("state")

backend/controllers/github_pull_request.go

@@ -9,7 +9,6 @@ import (
 	"runtime/debug"
 	"slices"
 	"strconv"
-	"strings"
 
 	"github.com/diggerhq/digger/backend/ci_backends"
 	config2 "github.com/diggerhq/digger/backend/config"
@@ -138,16 +137,27 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 			return nil
 		}
 
-		// Silently skip repos without digger.yml - this is expected for org-wide installations
-		if strings.Contains(err.Error(), "could not find digger.yml") ||
-			strings.Contains(err.Error(), "could not find digger.yaml") {
-			slog.Info("No Digger config found, skipping repo",
+		// Check if the error is due to missing digger config and the app is installed for all repos
+		if errors.Is(err, digger_config.ErrDiggerConfigNotFound) {
+			slog.Debug("Digger config not found, checking if app is installed for all repos",
 				"prNumber", prNumber,
 				"repoFullName", repoFullName,
 			)
-			return nil
+			isAllRepos, checkErr := utils.IsAllReposInstallation(appId, installationId)
+			if checkErr != nil {
+				slog.Warn("Failed to check if installation is for all repos",
+					"error", checkErr,
+				)
+			} else if isAllRepos {
+				slog.Info("Digger config not found but GitHub App is installed for all repos, skipping error comment",
+					"prNumber", prNumber,
+					"repoFullName", repoFullName,
+				)
+				return nil
+			}
 		}
 
+
 		slog.Error("Error getting Digger config for PR",
 			"prNumber", prNumber,
 			"repoFullName", repoFullName,
@@ -515,7 +525,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		commentReporterManager.UpdateComment(fmt.Sprintf(":x: Could not retrieve created batch: %v", err))
 		return fmt.Errorf("error getting digger batch")
 	}
-
+	
 	if config.CommentRenderMode == digger_config.CommentRenderModeGroupByModule {
 		slog.Info("Using GroupByModule render mode for comments", "prNumber", prNumber)
 

backend/controllers/projects.go

@@ -1026,7 +1026,6 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting refreshed batch"})
 		return
 	}
-	//err = UpdateCheckStatusForBatch(d.GithubClientProvider, refreshedBatch)
 	slog.Debug("Attempting to update GitHub Check Run for batch",
 		"batchId", batch.ID,
 		"checkRunId", refreshedBatch.CheckRunId,
@@ -1056,7 +1055,6 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting refreshed job"})
 		return
 	}
-	//err = UpdateCommitStatusForJob(d.GithubClientProvider, refreshedJob)
 	slog.Debug("Attempting to update GitHub Check Run for job",
 		"jobId", jobId,
 		"checkRunId", refreshedJob.CheckRunId,

backend/controllers/projects_helpers.go

@@ -210,9 +210,12 @@ func UpdateCheckRunForBatch(gh utils.GithubClientProvider, batch *models.DiggerB
 		return fmt.Errorf("error generating realtime comment message: %v", err)
 	}
 
-	summary, err := GenerateChecksSummaryForBatch(batch)
-	if err != nil {
-		slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err)
+	var summary = ""
+	if batch.Status == orchestrator_scheduler.BatchJobSucceeded || batch.Status == orchestrator_scheduler.BatchJobFailed {
+		summary, err = GenerateChecksSummaryForBatch(batch)
+		if err != nil {
+			slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err)
+		}
 	}
 
 	if isPlanBatch {
@@ -397,11 +400,15 @@ func UpdateCheckRunForJob(gh utils.GithubClientProvider, job *models.DiggerJob)
 		"```\n"
 
 
-	summary, err := GenerateChecksSummaryForJob(job)
-	if err != nil {
-		slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err)
+	var summary = ""
+	if job.Status == orchestrator_scheduler.DiggerJobSucceeded || job.Status == orchestrator_scheduler.DiggerJobFailed {
+		summary, err = GenerateChecksSummaryForJob(job)
+		if err != nil {
+			slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err)
+		}
 	}
 
+
 	slog.Debug("Updating PR status for job", "jobId", job.DiggerJobID, "status", status, "conclusion", conclusion)
 	if isPlan {
 		title := fmt.Sprintf("%v to create %v to update %v to delete", job.DiggerJobSummary.ResourcesCreated, job.DiggerJobSummary.ResourcesUpdated, job.DiggerJobSummary.ResourcesDeleted)

backend/models/scheduler.go

@@ -259,9 +259,9 @@ func GetCheckRunConclusionForJob(job *DiggerJob) (string, error) {
 		return "failure", nil
 	}
 	slog.Error("Unknown job status in GetCheckRunConclusionForJob - this will cause GitHub API 422 error",
-		"jobId", job.DiggerJobID,
-		"jobStatus", job.Status,
-		"jobStatusInt", int(job.Status),
-		"validStatuses", []string{"created", "triggered", "started", "queued_for_run", "succeeded", "failed"})
+	"jobId", job.DiggerJobID,
+	"jobStatus", job.Status,
+	"jobStatusInt", int(job.Status),
+	"validStatuses", []string{"created", "triggered", "started", "queued_for_run", "succeeded", "failed"})
 	return "", fmt.Errorf("unknown job status: %v", job.Status)
 }

docs/ce/local-development/backend.mdx

@@ -1,5 +1,5 @@
 ---
-title: Orchestrator local setup
+title: Backend (orchestrator) local setup
 ---
 
 The backend serves orchestration APIs, GitHub app endpoints, and internal APIs the UI relies on.
@@ -51,7 +51,7 @@ The backend serves orchestration APIs, GitHub app endpoints, and internal APIs t
 
 ## GitHub app integration
 
-- For a quick install link, set `ORCHESTRATOR_GITHUB_APP_URL` in `ui/.env.local` to your app's install URL (`https://github.com/apps/<app>/installations/new`).
+- For a quick install link, set `ORCHESTRATOR_GITHUB_APP_URL` in `ui/.env.local` to your app’s install URL (`https://github.com/apps/<app>/installations/new`).
 - To create a new app via the backend, open `http://localhost:3000/github/setup` (requires `HOSTNAME` set to a reachable URL for callbacks).
 
 ## Troubleshooting

docs/docs.json

@@ -223,4 +223,4 @@
       "linkedin": "https://www.linkedin.com/company/diggerhq/"
     }
   }
-}
+}

go.work.sum

@@ -1454,6 +1454,7 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
+github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
 github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
 github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw=
@@ -1622,6 +1623,7 @@ github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td
 github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
+github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg=
 github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
 github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
 github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
@@ -2031,6 +2033,7 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
 github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
+github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA=
 github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
 github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM=
 github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
@@ -2139,6 +2142,7 @@ github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 h1:BN/Nyn2nWMo
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY=
@@ -2179,7 +2183,6 @@ github.com/segmentio/conf v1.2.0 h1:5OT9+6OyVHLsFLsiJa/2KlqiA1m7mpdUBlkB/qYTMts=
 github.com/segmentio/conf v1.2.0/go.mod h1:Y3B9O/PqqWqjyxyWWseyj/quPEtMu1zDp/kVbSWWaB0=
 github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
-github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
 github.com/shirou/gopsutil/v3 v3.23.2 h1:PAWSuiAszn7IhPMBtXsbSCafej7PqUOvY6YywlQUExU=
 github.com/shirou/gopsutil/v3 v3.23.2/go.mod h1:gv0aQw33GLo3pG8SiWKiQrbDzbRY1K80RyZJ7V4Th1M=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=

libs/ci/github/github.go

@@ -490,55 +490,7 @@ func (svc GithubService) UpdateCheckRun(checkRunId string, options GithubCheckRu
 		opts.Conclusion = github.String(*conclusion)
 	}
 
-	checkRun, resp, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts)
-	
-	// Log rate limit information
-	if resp != nil {
-		limit := resp.Header.Get("X-RateLimit-Limit")
-		remaining := resp.Header.Get("X-RateLimit-Remaining")
-		reset := resp.Header.Get("X-RateLimit-Reset")
-		
-		if limit != "" && remaining != "" {
-			limitInt, _ := strconv.Atoi(limit)
-			remainingInt, _ := strconv.Atoi(remaining)
-			
-			// Calculate percentage remaining
-			var percentRemaining float64
-			if limitInt > 0 {
-				percentRemaining = (float64(remainingInt) / float64(limitInt)) * 100
-			}
-			
-			// Log based on severity
-			if remainingInt == 0 {
-				slog.Error("GitHub API rate limit EXHAUSTED",
-					"operation", "UpdateCheckRun",
-					"checkRunId", checkRunId,
-					"limit", limit,
-					"remaining", remaining,
-					"reset", reset,
-					"owner", owner,
-					"repo", repoName)
-			} else if percentRemaining < 20 {
-				slog.Warn("GitHub API rate limit getting LOW",
-					"operation", "UpdateCheckRun",
-					"checkRunId", checkRunId,
-					"limit", limit,
-					"remaining", remaining,
-					"percentRemaining", fmt.Sprintf("%.1f%%", percentRemaining),
-					"reset", reset,
-					"owner", owner,
-					"repo", repoName)
-			} else {
-				slog.Debug("GitHub API rate limit status",
-					"operation", "UpdateCheckRun",
-					"checkRunId", checkRunId,
-					"limit", limit,
-					"remaining", remaining,
-					"percentRemaining", fmt.Sprintf("%.1f%%", percentRemaining))
-			}
-		}
-	}
-	
+	checkRun, _, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts)
 	if err != nil {
 		slog.Error("Failed to update check run",
 			"inputCheckRunId", checkRunId,