Commit e64f44dc authored by Ivar Derksen's avatar Ivar Derksen
Browse files

Feat: run certificate and health checks async to speed up alerting time

parent c36d105d
package main
import (
"context"
"fmt"
"github.com/hashicorp/go-retryablehttp"
"io/ioutil"
"log"
"net/http"
"strings"
"sync"
"time"
)
type HealthCheck struct {
......@@ -20,48 +24,103 @@ type HealthCheck struct {
}
func runHealthChecks(checks []HealthCheck) (issues issueEntries) {
var waitGroup sync.WaitGroup
waitGroup.Add(len(checks))
issueChan := make(chan *issueEntry, len(checks))
for _, check := range checks {
log.Printf(" checking HTTP endpoint %s", check.RequestURL)
check := check
go func() {
issueChan <- runHealthCheck(check)
waitGroup.Done()
}()
// Introduce a small delay to prevent all checks to be started at the same time.
time.Sleep(10 * time.Millisecond)
}
// Set defaults
if check.RequestMethod == "" {
check.RequestMethod = "GET"
}
if check.ResponseStatusCodeEquals == 0 {
check.ResponseStatusCodeEquals = 200
}
waitGroup.Wait()
close(issueChan)
// Use retryablehttp to prevent false positives.
req, err := retryablehttp.NewRequest(check.RequestMethod, check.RequestURL, []byte(check.RequestBody))
if err != nil {
log.Printf("Health check %s: %s", check.RequestURL, err)
issues = append(issues, issueEntry{warning, fmt.Sprintf("%s: invalid health check", check.RequestURL)})
continue
}
for key, value := range check.RequestHeaders {
req.Header.Set(key, value)
for issue := range issueChan {
if issue != nil {
issues = append(issues, *issue)
}
}
return
}
func runHealthCheck(check HealthCheck) *issueEntry {
log.Printf(" checking HTTP endpoint %s", check.RequestURL)
// Set defaults
if check.RequestMethod == "" {
check.RequestMethod = "GET"
}
if check.ResponseStatusCodeEquals == 0 {
check.ResponseStatusCodeEquals = 200
}
// Use retryablehttp to prevent false positives.
req, err := retryablehttp.NewRequest(check.RequestMethod, check.RequestURL, []byte(check.RequestBody))
if err != nil {
log.Printf("Health check %s: %s", check.RequestURL, err)
return &issueEntry{warning, fmt.Sprintf("%s: invalid health check", check.RequestURL)}
}
for key, value := range check.RequestHeaders {
req.Header.Set(key, value)
}
var intermediateIssue *issueEntry
resp, err := retryablehttp.NewClient().Do(req)
if err != nil {
issues = append(issues, issueEntry{danger, fmt.Sprintf("%s: cannot be reached", check.RequestURL)})
continue
client := retryablehttp.NewClient()
client.HTTPClient.Timeout = 3 * time.Second
client.CheckRetry = func(ctx context.Context, resp *http.Response, respErr error) (bool, error) {
retry, err2 := retryablehttp.DefaultRetryPolicy(ctx, resp, respErr)
if !retry {
return false, err2
}
if resp.StatusCode != check.ResponseStatusCodeEquals {
issues = append(issues, issueEntry{danger, fmt.Sprintf("%s: received unexpected status code %d", check.RequestURL, resp.StatusCode)})
continue
if intermediateIssue == nil {
intermediateIssue = generateHealthCheckIssueEntry(check, resp, respErr)
}
return true, err2
}
for key, value := range check.ResponseHeaderContains {
if resp.Header.Get(key) != value {
issues = append(issues, issueEntry{danger, fmt.Sprintf("%s: expected response header \"%s: %s\" could not be found", check.RequestURL, key, value)})
}
resp, err := client.Do(req)
issue := generateHealthCheckIssueEntry(check, resp, err)
if issue != nil {
return issue
}
// Generate warning if health check was unstable.
if intermediateIssue != nil {
return &issueEntry{
issueType: warning,
message: fmt.Sprintf("Unstable health check: %s", intermediateIssue.message),
}
}
return nil
}
respBody, err := ioutil.ReadAll(resp.Body)
if !strings.Contains(string(respBody), check.ResponseBodyContains) {
issues = append(issues, issueEntry{danger, fmt.Sprintf("%s: expected response body \"%s\" could not be found", check.RequestURL, check.ResponseBodyContains)})
func generateHealthCheckIssueEntry(check HealthCheck, resp *http.Response, respErr error) *issueEntry {
if respErr != nil {
return &issueEntry{danger, fmt.Sprintf("%s: cannot be reached", check.RequestURL)}
}
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return &issueEntry{danger, fmt.Sprintf("%s: response body could not be read", check.RequestURL)}
}
if resp.StatusCode != check.ResponseStatusCodeEquals {
return &issueEntry{danger, fmt.Sprintf("%s: received unexpected status code %d", check.RequestURL, resp.StatusCode)}
}
for key, value := range check.ResponseHeaderContains {
if resp.Header.Get(key) != value {
return &issueEntry{danger, fmt.Sprintf("%s: expected response header \"%s: %s\" could not be found", check.RequestURL, key, value)}
}
}
return
if !strings.Contains(string(respBody), check.ResponseBodyContains) {
return &issueEntry{danger, fmt.Sprintf("%s: expected response body \"%s\" could not be found", check.RequestURL, check.ResponseBodyContains)}
}
return nil
}
......@@ -17,6 +17,7 @@ import (
"os"
"path"
"strings"
"sync"
"time"
"github.com/hashicorp/go-retryablehttp"
......@@ -342,16 +343,36 @@ func logCurrentIssues(curIssues []string) {
}
func checkCertificateExpiry() (ret issueEntries) {
for _, url := range conf.CheckCertificateExpiry {
log.Printf(" checking certificate expiry on %s", url)
ret = append(ret, checkCertificateExpiryOf(url)...)
var waitGroup sync.WaitGroup
waitGroup.Add(len(conf.CheckCertificateExpiry))
issueEntriesChan := make(chan issueEntries, len(conf.CheckCertificateExpiry))
for _, check := range conf.CheckCertificateExpiry {
check := check
go func() {
issueEntriesChan <- checkCertificateExpiryOf(check)
waitGroup.Done()
}()
// Introduce a small delay to prevent all checks to be started at the same time.
time.Sleep(10 * time.Millisecond)
}
waitGroup.Wait()
close(issueEntriesChan)
for entries := range issueEntriesChan {
ret = append(ret, entries...)
}
return
}
func checkCertificateExpiryOf(url string) (ret issueEntries) {
log.Printf(" checking certificate expiry on %s", url)
// Use retryablehttp to prevent false positives.
resp, err := retryablehttp.Head(url)
client := retryablehttp.NewClient()
client.HTTPClient.Timeout = 3 * time.Second
resp, err := client.Head(url)
if err != nil {
ret = append(ret, issueEntry{danger, fmt.Sprintf("%s: error %s", url, err)})
return
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment