README.md

@@ -2,58 +2,41 @@
 
 _A round-robin load balancer for HTTP proxies._
 
-This is a simple proxy load balancer that will route requests to a cluster of proxy backends in a round-robin fashion.
+This is a simple proxy load balancer that will route requests to a cluster of proxy backends in a round-robin fashion. This makes it easy to connect your clients to a large number of proxy servers without worrying about implementing anything special clientside.
-This makes it easy to connect your clients to a large number of proxy servers without worrying about implementing
-anything special client-side.
 
-This proxy server will transparently forward HTTPS requests without terminating them, meaning a self-signed certificate
+This proxy server will transparently forward HTTPS requests without terminating them, meaning a self-signed certificate is not required. Downstream HTTPS proxy servers are not supported.
-is not required. Downstream HTTPS proxy servers are not supported.
 
 Memory usage sits at around 25M under load.
 
 ## Install
 
-1. Download the latest release from [/releases](https://git.evulid.cc/cyberes/proxy-loadbalancer/releases) or
+1.  Download the latest release from [/releases](https://git.evulid.cc/cyberes/proxy-loadbalancer/releases) or run `./build.sh` to build the program locally.
-   run `./build.sh` to build the program locally.
+2.  `cp config.example.yml config.yml`
-2. `cp config.example.yml config.yml`
+3.  Edit the config.
-3. Edit the config.
+4.  Start the loadbalancer with `./proxy-loadbalancer --config [path to your config.yml]`
-4. Start the loadbalancer with `./proxy-loadbalancer --config [path to your config.yml]`
-
-The load balancer has experimental support for using [curl-impersonate](https://github.com/lwthiker/curl-impersonate) to masquerade as the Chrome browser when
-performing proxy checks.
-
-1. Download `*.x86_64-linux-gnu.tar.gz ` from https://github.com/lwthiker/curl-impersonate/releases
-2. Set `proxy_check_impersonate_chrome: true`
-3. Enter the path to the `curl_chrome116` binary in `proxy_check_impersonate_chrome_binary`
 
 ## Use
 
-You can run your own "public IP delivery server" `canihazip` <https://git.evulid.cc/cyberes/canihazip> or use the
+You can run your own "public IP delivery server" `canihazip` <https://git.evulid.cc/cyberes/canihazip> or use the default `api.ipify.org`
-default `api.ipify.org`
 
 An example systemd service `loadbalancer.service` is provided.
 
 The server displays health, stats, info at `/json`.
 
-Use `--log-third-party-test-failures` along with `--debug` when you want extra info on the third-party proxy tests. This
-can get very noisy if you have lots of third-party proxies so it's hidden behind an extra flag.
-
 ```
 === Proxy Load Balancer ===
-Usage of /tmp/go-build1714785557/b001/exe/proxy-loadbalancer:
+Usage of ./proxy-loadbalancer:
   --config [string]
-        Path to the config file
+               Path to the config file
-  -d, --debug
+  -d, --debug  
-        Enable debug mode
+               Enable debug mode
-  -l, --log-third-party-test-failures
+  --v          Print version and exit
-        Log third-party test debug info
+  -h, --help   Print this help message
-  -v    Print version and exit
 ```
 
 ## Special Headers
 
 The load balancer accepts special headers to control its behavior:
 
-- `Thirdparty-Bypass`: don't use any third-party endpoints for this request.
+-   `Thirdparty-Bypass`: don't use any third-party endpoints for this request.
-- `Thirdparty-Include-Broken`: use all online endpoints for this request, including third-party ones that failed the
+-   `Thirdparty-Include-Broken`: use all online endpoints for this request, including third-party ones that failed the special test.
-  special test.

config.example.yml

@@ -1,25 +1,17 @@
 # Port to run on.
-http_port:                             9000
+http_port:                 9000
 
 # How many proxies will be checked at once?
-proxy_checkers:                        50
+proxy_checkers:            50
 
 # The interval between proxy checks in seconds.
-proxy_check_interval:                  60
+proxy_check_interval:      60
 
 # URL to get a proxy's IP.
-ip_checker_url:                        https://api.ipify.org
+ip_checker_url:            https://api.ipify.org
 
 # Connection timeout for the proxies in seconds.
-proxy_connect_timeout:                 60
+proxy_connect_timeout:     60
-
-# How many times to retry a proxy connection.
-# On each retry a new proxy will be chosen.
-proxy_connect_retries:                   3
-
-# Use `curl-impersonate` to pretend to be Chrome when testing proxies.
-proxy_check_impersonate_chrome:        false
-proxy_check_impersonate_chrome_binary: ./curl_chrome116
 
 # Your proxies.
 proxy_pool_ours:
@@ -42,7 +34,7 @@ thirdparty_bypass_domains:
 
 # Shuffle the proxy lists whenever the background thread refreshes them.
 # If false, round-robin on default order.
-shuffle_proxies:                       false
+shuffle_proxies:           false
 
 # Don't allow requests to these domains through the proxy.
 blocked_domains:
@@ -51,4 +43,4 @@ blocked_domains:
 # Resolve specific domains through specific proxies.
 # Proxies here are not validated.
 resolve_through:
-  github.com:                          http://1.2.3.4:3128
+  github.com:              http://1.2.3.4:3128

requirements.txt

@@ -0,0 +1,7 @@
+proxy.py @ git+https://github.com/abhinavsingh/proxy.py.git@develop
+redis
+requests
+coloredlogs
+psutil
+flask
+async_timeout

src/config/config.go

@@ -2,10 +2,7 @@ package config
 
 import (
 	"errors"
-	"fmt"
 	"github.com/spf13/viper"
-	"os"
-	"os/exec"
 	"time"
 )
 
@@ -13,21 +10,18 @@ import (
 var cfg *Config
 
 type Config struct {
-	HTTPPort                          string
+	HTTPPort                string
-	IpCheckerURL                      string
+	IpCheckerURL            string
-	MaxProxyCheckers                  int
+	MaxProxyCheckers        int
-	ProxyConnectTimeout               time.Duration
+	ProxyConnectTimeout     time.Duration
-	ProxyPoolOurs                     []string
+	ProxyPoolOurs           []string
-	ProxyPoolThirdparty               []string
+	ProxyPoolThirdparty     []string
-	ThirdpartyTestUrls                []string
+	ThirdpartyTestUrls      []string
-	ThirdpartyBypassDomains           []string
+	ThirdpartyBypassDomains []string
-	ShuffleProxies                    bool
+	ShuffleProxies          bool
-	BlockedDomains                    []string
+	BlockedDomains          []string
-	ResolveThrough                    map[string]string
+	ResolveThrough          map[string]string
-	ProxyCheckInterval                int
+	ProxyCheckInterval      int
-	ProxyCheckImpersonateChrome       bool
-	ProxyCheckImpersonateChromeBinary string
-	ProxyConnectRetries               int
 }
 
 func SetConfig(configFile string) (*Config, error) {
@@ -48,9 +42,6 @@ func SetConfig(configFile string) (*Config, error) {
 	viper.SetDefault("blocked_domains", make([]string, 0))
 	viper.SetDefault("resolve_through", make(map[string]string))
 	viper.SetDefault("proxy_check_interval", 60)
-	viper.SetDefault("proxy_check_impersonate_chrome", false)
-	viper.SetDefault("proxy_check_impersonate_chrome_binary", nil)
-	viper.SetDefault("proxy_connect_retries", 3)
 
 	err := viper.ReadInConfig()
 	if err != nil {
@@ -58,20 +49,17 @@ func SetConfig(configFile string) (*Config, error) {
 	}
 
 	config := &Config{
-		HTTPPort:                          viper.GetString("http_port"),
+		HTTPPort:                viper.GetString("http_port"),
-		IpCheckerURL:                      viper.GetString("ip_checker_url"),
+		IpCheckerURL:            viper.GetString("ip_checker_url"),
-		MaxProxyCheckers:                  viper.GetInt("proxy_checkers"),
+		MaxProxyCheckers:        viper.GetInt("proxy_checkers"),
-		ProxyPoolOurs:                     viper.GetStringSlice("proxy_pool_ours"),
+		ProxyPoolOurs:           viper.GetStringSlice("proxy_pool_ours"),
-		ProxyPoolThirdparty:               viper.GetStringSlice("proxy_pool_thirdparty"),
+		ProxyPoolThirdparty:     viper.GetStringSlice("proxy_pool_thirdparty"),
-		ThirdpartyTestUrls:                viper.GetStringSlice("thirdparty_test_urls"),
+		ThirdpartyTestUrls:      viper.GetStringSlice("thirdparty_test_urls"),
-		ThirdpartyBypassDomains:           viper.GetStringSlice("thirdparty_bypass_domains"),
+		ThirdpartyBypassDomains: viper.GetStringSlice("thirdparty_bypass_domains"),
-		ShuffleProxies:                    viper.GetBool("shuffle_proxies"),
+		ShuffleProxies:          viper.GetBool("shuffle_proxies"),
-		BlockedDomains:                    viper.GetStringSlice("blocked_domains"),
+		BlockedDomains:          viper.GetStringSlice("blocked_domains"),
-		ResolveThrough:                    viper.GetStringMapString("resolve_through"),
+		ResolveThrough:          viper.GetStringMapString("resolve_through"),
-		ProxyCheckInterval:                viper.GetInt("proxy_check_interval"),
+		ProxyCheckInterval:      viper.GetInt("proxy_check_interval"),
-		ProxyCheckImpersonateChrome:       viper.GetBool("proxy_check_impersonate_chrome"),
-		ProxyCheckImpersonateChromeBinary: viper.GetString("proxy_check_impersonate_chrome_binary"),
-		ProxyConnectRetries:               viper.GetInt("proxy_connect_retries"),
 	}
 
 	if len(config.ProxyPoolOurs) == 0 && len(config.ProxyPoolThirdparty) == 0 {
@@ -98,21 +86,6 @@ func SetConfig(configFile string) (*Config, error) {
 		return nil, proxyPoolThirdpartyErr
 	}
 
-	if config.ProxyCheckImpersonateChrome {
-		if _, err := os.Stat(config.ProxyCheckImpersonateChromeBinary); os.IsNotExist(err) {
-			return nil, errors.New(fmt.Sprintf(`curl-impersonate-chrome binary does not exist: "%s"`, config.ProxyCheckImpersonateChromeBinary))
-		}
-		cmd := exec.Command(config.ProxyCheckImpersonateChromeBinary, "--help")
-		err = cmd.Run()
-		if err != nil {
-			return nil, errors.New(fmt.Sprintf(`curl-impersonate-chrome binary failed to run: %s`, err))
-		}
-	}
-
-	if config.ProxyConnectRetries <= 0 {
-		return nil, errors.New("proxy_connect_retries must be greater than 0")
-	}
-
 	cfg = config
 	return config, nil
 }

src/config/flags.go

@@ -1,27 +0,0 @@
-package config
-
-import "flag"
-
-var CliArgs *CliConfig
-
-type CliConfig struct {
-	ConfigFile        string
-	Debug             bool
-	Help              bool
-	Version           bool
-	LogThirdPartyTest bool
-}
-
-func ParseArgs() {
-	if CliArgs != nil {
-		panic("already defined")
-	}
-	CliArgs = &CliConfig{}
-	flag.StringVar(&CliArgs.ConfigFile, "config", "", "Path to the config file")
-	flag.BoolVar(&CliArgs.Debug, "d", false, "Enable debug mode")
-	flag.BoolVar(&CliArgs.Debug, "debug", false, "Enable debug mode")
-	flag.BoolVar(&CliArgs.Debug, "l", false, "Log third-party test debug info")
-	flag.BoolVar(&CliArgs.Debug, "log-third-party-test-failures", false, "Log third-party test debug info")
-	flag.BoolVar(&CliArgs.Version, "v", false, "Print version and exit")
-	flag.Parse()
-}

src/proxy-loadbalancer.go

@@ -13,17 +13,26 @@ import (
 	"runtime/debug"
 )
 
+type cliConfig struct {
+	configFile         string
+	initialCrawl       bool
+	debug              bool
+	disableElasticSync bool
+	help               bool
+	version            bool
+}
+
 var Version = "development"
 var VersionDate = "not set"
 
 func main() {
 	fmt.Println("=== Proxy Load Balancer ===")
-	config.ParseArgs()
+	cliArgs := parseArgs()
-	if config.CliArgs.Help {
+	if cliArgs.help {
 		flag.Usage()
 		os.Exit(0)
 	}
-	if config.CliArgs.Version {
+	if cliArgs.version {
 		buildInfo, ok := debug.ReadBuildInfo()
 
 		if ok {
@@ -42,7 +51,7 @@ func main() {
 		os.Exit(0)
 	}
 
-	if config.CliArgs.Debug {
+	if cliArgs.debug {
 		logging.InitLogger(logrus.DebugLevel)
 	} else {
 		logging.InitLogger(logrus.InfoLevel)
@@ -50,7 +59,7 @@ func main() {
 	log := logging.GetLogger()
 	log.Debugln("Initializing...")
 
-	if config.CliArgs.ConfigFile == "" {
+	if cliArgs.configFile == "" {
 		exePath, err := os.Executable()
 		if err != nil {
 			panic(err)
@@ -61,24 +70,20 @@ func main() {
 			if _, err := os.Stat(filepath.Join(exeDir, "config.yaml")); err == nil {
 				log.Fatalln("Both config.yml and config.yaml exist in the executable directory. Please specify one with the --config flag.")
 			}
-			config.CliArgs.ConfigFile = filepath.Join(exeDir, "config.yml")
+			cliArgs.configFile = filepath.Join(exeDir, "config.yml")
 		} else if _, err := os.Stat(filepath.Join(exeDir, "config.yaml")); err == nil {
-			config.CliArgs.ConfigFile = filepath.Join(exeDir, "config.yaml")
+			cliArgs.configFile = filepath.Join(exeDir, "config.yaml")
 		} else {
 			log.Fatalln("No config file found in the executable directory. Please provide one with the --config flag.")
 		}
 	}
-	configData, err := config.SetConfig(config.CliArgs.ConfigFile)
+	configData, err := config.SetConfig(cliArgs.configFile)
 	if err != nil {
 		log.Fatalf(`Failed to load config: %s`, err)
 	}
 
 	log.Debugf(`Proxy check interval: %d sec`, config.GetConfig().ProxyCheckInterval)
 
-	if config.GetConfig().ProxyCheckImpersonateChrome {
-		log.Debugf(`Using curl-impersonate binary: %s`, config.GetConfig().ProxyCheckImpersonateChromeBinary)
-	}
-
 	proxyCluster := proxy.NewForwardProxyCluster()
 	go func() {
 		log.Fatal(http.ListenAndServe(":"+configData.HTTPPort, proxyCluster))
@@ -91,3 +96,13 @@ func main() {
 
 	select {}
 }
+
+func parseArgs() cliConfig {
+	var cliArgs cliConfig
+	flag.StringVar(&cliArgs.configFile, "config", "", "Path to the config file")
+	flag.BoolVar(&cliArgs.debug, "d", false, "Enable debug mode")
+	flag.BoolVar(&cliArgs.debug, "debug", false, "Enable debug mode")
+	flag.BoolVar(&cliArgs.version, "v", false, "Print version and exit")
+	flag.Parse()
+	return cliArgs
+}

src/proxy/checkProxy.go

@@ -1,17 +1,11 @@
 package proxy
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"main/config"
 	"net/http"
 	"net/url"
-	"os"
-	"os/exec"
-	"regexp"
-	"strconv"
-	"strings"
 )
 
 func sendRequestThroughProxy(pxy string, targetURL string) (string, error) {
@@ -23,128 +17,43 @@ func sendRequestThroughProxy(pxy string, targetURL string) (string, error) {
 	if proxyUser != "" && proxyPass != "" {
 		parsedProxyUrl.User = url.UserPassword(proxyUser, proxyPass)
 	}
-
+	transport := &http.Transport{
-	if !config.GetConfig().ProxyCheckImpersonateChrome {
+		Proxy: http.ProxyURL(parsedProxyUrl),
-		transport := &http.Transport{
-			Proxy: http.ProxyURL(parsedProxyUrl),
-		}
-		client := &http.Client{
-			Transport: transport,
-			Timeout:   config.GetConfig().ProxyConnectTimeout,
-		}
-
-		req, err := http.NewRequest("GET", targetURL, nil)
-		if err != nil {
-			return "", err
-		}
-
-		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
-		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
-		req.Header.Set("Priority", "u=0, i")
-		req.Header.Set("Sec-Ch-Ua", `Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"`)
-		req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
-		req.Header.Set("Sec-Ch-Ua-Platform", `"Windows"`)
-		req.Header.Set("Sec-Fetch-Dest", "document")
-		req.Header.Set("Sec-Fetch-Mode", "navigate")
-		req.Header.Set("Sec-Fetch-Site", "none")
-		req.Header.Set("Sec-Fetch-User", "?1")
-		req.Header.Set("Upgrade-Insecure-Requests", "1")
-		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36")
-
-		resp, err := client.Do(req)
-		if err != nil {
-			return "", err
-		}
-		defer resp.Body.Close()
-		if resp.StatusCode == http.StatusOK {
-			bodyBytes, err := io.ReadAll(resp.Body)
-			if err != nil {
-				return "", err
-			}
-			return string(bodyBytes), nil
-		}
-		return "", fmt.Errorf("bad response code %d", resp.StatusCode)
-	} else {
-		tmpfile, err := os.CreateTemp("", "response")
-		if err != nil {
-			log.Fatal(err)
-		}
-		defer os.Remove(tmpfile.Name()) // clean up
-
-		cmd := exec.Command(
-			config.GetConfig().ProxyCheckImpersonateChromeBinary,
-			"--proxy",
-			parsedProxyUrl.String(),
-			"-o",
-			tmpfile.Name(),
-			"-w",
-			"%{http_code}",
-			"--ciphers",
-			"TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA",
-			"-H",
-			`sec-ch-ua: "Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"`,
-			"-H",
-			`sec-ch-ua-mobile: ?0`,
-			"-H",
-			`sec-ch-ua-platform: "Windows"`,
-			"-H",
-			`Upgrade-Insecure-Requests: 1`,
-			"-H",
-			`User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36`,
-			"-H",
-			`Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7`,
-			"-H",
-			`Sec-Fetch-Site: none`,
-			"-H",
-			`Sec-Fetch-Mode: navigate`,
-			"-H",
-			`Sec-Fetch-User: ?1`,
-			"-H",
-			`Sec-Fetch-Dest: document`,
-			"-H",
-			`Accept-Encoding: gzip, deflate, br`,
-			"-H",
-			`Accept-Language: en-US,en;q=0.9`,
-			"--http2",
-			"--http2-no-server-push",
-			"--compressed",
-			"--tlsv1.2",
-			"--alps",
-			"--tls-permute-extensions",
-			"--cert-compression",
-			"brotli",
-			targetURL,
-		)
-
-		output, err := cmd.Output()
-		if err != nil {
-			var exitError *exec.ExitError
-			if errors.As(err, &exitError) {
-				return "", fmt.Errorf("command exited with code %d: %s", exitError.ExitCode(), exitError.Stderr)
-			}
-			return "", err
-		}
-
-		outputStr := strings.TrimSpace(string(output))
-		match, _ := regexp.MatchString(`^\d{3}$`, outputStr)
-		if !match {
-			return "", fmt.Errorf("unexpected output from curl command: %s", outputStr)
-		}
-
-		statusCode, err := strconv.Atoi(outputStr)
-		if err != nil {
-			return "", err
-		}
-
-		if statusCode != http.StatusOK {
-			return "", fmt.Errorf("bad response code %d", statusCode)
-		}
-
-		body, err := os.ReadFile(tmpfile.Name())
-		if err != nil {
-			return "", err
-		}
-
-		return string(body), nil
 	}
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   config.GetConfig().ProxyConnectTimeout,
+	}
+
+	req, err := http.NewRequest("GET", targetURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	req.Header.Set("Priority", "u=0, i")
+	req.Header.Set("Sec-Ch-Ua", `"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"`)
+	req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
+	req.Header.Set("Sec-Ch-Ua-Platform", `"Windows"`)
+	req.Header.Set("Sec-Fetch-Dest", "document")
+	req.Header.Set("Sec-Fetch-Mode", "navigate")
+	req.Header.Set("Sec-Fetch-Site", "cross-site")
+	req.Header.Set("Sec-Fetch-User", "?1")
+	req.Header.Set("Upgrade-Insecure-Requests", "1")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusOK {
+		bodyBytes, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return "", err
+		}
+		return string(bodyBytes), nil
+	}
+	return "", fmt.Errorf("bad response code %d", resp.StatusCode)
 }

src/proxy/handleConnect.go

@@ -19,13 +19,8 @@ var (
 	HeaderThirdpartyBypass        = "Thirdparty-Bypass"
 )
 
-func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time, errorMsg *string) {
+func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time) {
-	msg := fmt.Sprintf(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds())
+	log.Infof(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds())
-	if *errorMsg == "" {
-		log.Infof(msg)
-	} else {
-		log.Errorf(`%s -- %s`, msg, *errorMsg)
-	}
 }
 
 func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter, req *http.Request) (string, string, string, string, *url.URL, error) {
@@ -51,11 +46,11 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
 		return "", "", "", "", nil, errors.New(errStr)
 	}
 
-	headerIncludeBrokenThirdparty := req.Header.Get(HeaderThirdpartyIncludeBroken) != ""
+	headerIncludeBrokenThirdparty := req.Header.Get(HeaderThirdpartyIncludeBroken)
 	req.Header.Del(HeaderThirdpartyIncludeBroken)
-	headerBypassThirdparty := req.Header.Get(HeaderThirdpartyBypass) != ""
+	headerBypassThirdparty := req.Header.Get(HeaderThirdpartyBypass)
 	req.Header.Del(HeaderThirdpartyBypass)
-	if headerBypassThirdparty && headerIncludeBrokenThirdparty {
+	if headerBypassThirdparty != "" && headerIncludeBrokenThirdparty != "" {
 		errStr := "duplicate options headers detected, rejecting request"
 		http.Error(w, errStr, http.StatusBadRequest)
 		return "", "", "", "", nil, errors.New(errStr)
@@ -69,9 +64,9 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
 		if slices.Contains(config.GetConfig().ThirdpartyBypassDomains, urlHostname) {
 			selectedProxy = p.getProxyFromOurs()
 		} else {
-			if headerIncludeBrokenThirdparty {
+			if headerIncludeBrokenThirdparty != "" {
 				selectedProxy = p.getProxyFromAllWithBroken()
-			} else if headerBypassThirdparty {
+			} else if headerBypassThirdparty != "" {
 				selectedProxy = p.getProxyFromOurs()
 			} else {
 				selectedProxy = p.getProxyFromAll()
@@ -90,6 +85,7 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
 	}
 
 	return selectedProxy, proxyUser, proxyPass, proxyHost, parsedProxyUrl, nil
+
 }
 
 func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.Request) {
@@ -97,20 +93,17 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
 	remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
 	_, proxyUser, proxyPass, proxyHost, parsedProxyUrl, err := p.validateRequestAndGetProxy(w, req)
 	if err != nil {
+		// Error has already been handled, just log and return.
 		if proxyHost == "" {
 			proxyHost = "none"
 		}
 		log.Debugf(`%s -> %s -> %s -- HTTP -- Rejecting request: %s`, remoteAddr, proxyHost, req.Host, err)
 		return
 	}
-
 	var returnCode *int
 	returnCode = new(int)
 	*returnCode = -1
-	var errorMsg *string
+	defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime)
-	errorMsg = new(string)
-	*errorMsg = ""
-	defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime, errorMsg)
 
 	parsedProxyUrl.Scheme = "http"
 	if proxyUser != "" && proxyPass != "" {
@@ -125,7 +118,7 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
 
 	proxyReq, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
 	if err != nil {
-		*errorMsg = fmt.Sprintf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err)
+		log.Errorf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err)
 		http.Error(w, "failed to make request to downstream", http.StatusInternalServerError)
 		return
 	}
@@ -133,25 +126,18 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
 	copyHeader(proxyReq.Header, req.Header)
 	proxyReq.Header.Set("X-Forwarded-For", req.RemoteAddr)
 
-	for i := 0; i < config.GetConfig().ProxyConnectRetries; i++ { // Retry mechanic
+	resp, err := client.Do(proxyReq)
-		resp, err := client.Do(proxyReq)
+	if err != nil {
-		if err != nil {
+		log.Errorf(`Failed to execute %s request to "%s": %s`, req.Method, req.URL.String(), err)
-			*errorMsg = fmt.Sprintf(`Failed to execute %s request to "%s" - attempt %d/%d - %s`, req.Method, req.URL.String(), i+1, config.GetConfig().ProxyConnectRetries, err)
+		http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
-			if i < config.GetConfig().ProxyConnectRetries-1 {
+		return
-				continue
-			} else {
-				http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
-				return
-			}
-		} else {
-			defer resp.Body.Close()
-			*returnCode = resp.StatusCode
-			copyHeader(w.Header(), resp.Header)
-			w.WriteHeader(resp.StatusCode)
-			io.Copy(w, resp.Body)
-			break
-		}
 	}
+	defer resp.Body.Close()
+	*returnCode = resp.StatusCode
+
+	copyHeader(w.Header(), resp.Header)
+	w.WriteHeader(resp.StatusCode)
+	io.Copy(w, resp.Body)
 }
 
 func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http.Request) {
@@ -160,6 +146,7 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
 	targetHost, _, _ := net.SplitHostPort(req.Host)
 	_, proxyUser, proxyPass, proxyHost, _, err := p.validateRequestAndGetProxy(w, req)
 	if err != nil {
+		// Error has already been handled, just log and return.
 		if proxyHost == "" {
 			proxyHost = "none"
 		}
@@ -169,26 +156,14 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
 	var returnCode *int
 	returnCode = new(int)
 	*returnCode = -1
-	var errorMsg *string
+	defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime)
-	errorMsg = new(string)
-	*errorMsg = ""
-	defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime, errorMsg)
 
-	var proxyConn net.Conn
+	// Start a connection to the downstream proxy server.
-	for i := 0; i < config.GetConfig().ProxyConnectRetries; i++ {
+	proxyConn, err := net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout)
-		// Start a connection to the downstream proxy server.
+	if err != nil {
-		proxyConn, err = net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout)
+		log.Errorf(`Failed to dial proxy %s - %s`, proxyHost, err)
-		if err != nil {
+		http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable)
-			*errorMsg = fmt.Sprintf(`Failed to dial proxy %s - attempt %d/%d - %s`, proxyHost, i+1, config.GetConfig().ProxyConnectRetries, err)
+		return
-			if i < config.GetConfig().ProxyConnectRetries-1 {
-				continue
-			} else {
-				http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable)
-				return
-			}
-		} else {
-			break
-		}
 	}
 
 	// Proxy authentication
@@ -203,7 +178,7 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
 	}
 	resp, err := http.ReadResponse(bufio.NewReader(proxyConn), req)
 	if resp == nil {
-		*errorMsg = fmt.Sprintf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
+		log.Errorf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
 		http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
 		return
 	} else if err != nil {
@@ -214,14 +189,14 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
 	w.WriteHeader(http.StatusOK)
 	hj, ok := w.(http.Hijacker)
 	if !ok {
-		*errorMsg = fmt.Sprintf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
+		log.Errorf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
 		http.Error(w, "failed to forward connection to downstream", http.StatusServiceUnavailable)
 		return
 	}
 
 	clientConn, _, err := hj.Hijack()
 	if err != nil {
-		*errorMsg = fmt.Sprintf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
+		log.Errorf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
 		http.Error(w, "failed to execute connection forwarding to downstream", http.StatusServiceUnavailable)
 		return
 	}

src/proxy/threads.go

@@ -10,8 +10,6 @@ import (
 	"time"
 )
 
-// TODO: fix 503 errors returned during proxy checking process
-
 func (p *ForwardProxyCluster) ValidateProxiesThread() {
 	log.Infoln("Doing initial backend check, please wait...")
 	started := false
@@ -19,7 +17,6 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
 	ctx := context.TODO()
 
 	for {
-		startTime := time.Now()
 		p.refreshInProgress = true
 		allProxies := removeDuplicates(append(config.GetConfig().ProxyPoolOurs, config.GetConfig().ProxyPoolThirdparty...))
 		newOurOnlineProxies := make([]string, 0)
@@ -60,13 +57,10 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
 				// Test the proxy.
 				ipAddr, testErr := sendRequestThroughProxy(pxy, config.GetConfig().IpCheckerURL)
 				if testErr != nil {
+					log.Warnf("Validate - %s failed: %s", proxyHost, testErr)
 					if isThirdparty(pxy) {
-						if config.CliArgs.LogThirdPartyTest {
-							log.Debugf("Validate - %s failed: %s", proxyHost, testErr)
-						}
 						newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
 					} else {
-						log.Debugf("Validate - %s failed: %s", proxyHost, testErr)
 						newOurOfflineProxies = append(newOurOfflineProxies, pxy)
 					}
 					return
@@ -90,9 +84,7 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
 						if bv3hiErr != nil {
 							okToAdd = false
 							newThirdpartyBrokenProxies = append(newThirdpartyBrokenProxies, pxy)
-							if config.CliArgs.LogThirdPartyTest {
+							log.Debugf("Validate - %s failed third-party test: %s", proxyHost, bv3hiErr)
-								log.Debugf(`%s failed third-party test for URL "%s" -- %s`, proxyHost, d, bv3hiErr)
-							}
 							break
 						}
 					}
@@ -129,8 +121,8 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
 		}
 
 		p.mu.RLock()
-		log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d, Elapsed: %s",
+		log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d",
-			len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+len(p.thirdpartyOnlineProxies), time.Since(startTime),
+			len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+len(p.thirdpartyOnlineProxies),
 		)
 		p.mu.RUnlock()