Compare commits

..

8 Commits

9 changed files with 330 additions and 149 deletions

View File

@ -2,41 +2,58 @@
_A round-robin load balancer for HTTP proxies._ _A round-robin load balancer for HTTP proxies._
This is a simple proxy load balancer that will route requests to a cluster of proxy backends in a round-robin fashion. This makes it easy to connect your clients to a large number of proxy servers without worrying about implementing anything special clientside. This is a simple proxy load balancer that will route requests to a cluster of proxy backends in a round-robin fashion.
This makes it easy to connect your clients to a large number of proxy servers without worrying about implementing
anything special client-side.
This proxy server will transparently forward HTTPS requests without terminating them, meaning a self-signed certificate is not required. Downstream HTTPS proxy servers are not supported. This proxy server will transparently forward HTTPS requests without terminating them, meaning a self-signed certificate
is not required. Downstream HTTPS proxy servers are not supported.
Memory usage sits at around 25M under load. Memory usage sits at around 25M under load.
## Install ## Install
1. Download the latest release from [/releases](https://git.evulid.cc/cyberes/proxy-loadbalancer/releases) or run `./build.sh` to build the program locally. 1. Download the latest release from [/releases](https://git.evulid.cc/cyberes/proxy-loadbalancer/releases) or
2. `cp config.example.yml config.yml` run `./build.sh` to build the program locally.
3. Edit the config. 2. `cp config.example.yml config.yml`
4. Start the loadbalancer with `./proxy-loadbalancer --config [path to your config.yml]` 3. Edit the config.
4. Start the loadbalancer with `./proxy-loadbalancer --config [path to your config.yml]`
The load balancer has experimental support for using [curl-impersonate](https://github.com/lwthiker/curl-impersonate) to masquerade as the Chrome browser when
performing proxy checks.
1. Download `*.x86_64-linux-gnu.tar.gz ` from https://github.com/lwthiker/curl-impersonate/releases
2. Set `proxy_check_impersonate_chrome: true`
3. Enter the path to the `curl_chrome116` binary in `proxy_check_impersonate_chrome_binary`
## Use ## Use
You can run your own "public IP delivery server" `canihazip` <https://git.evulid.cc/cyberes/canihazip> or use the default `api.ipify.org` You can run your own "public IP delivery server" `canihazip` <https://git.evulid.cc/cyberes/canihazip> or use the
default `api.ipify.org`
An example systemd service `loadbalancer.service` is provided. An example systemd service `loadbalancer.service` is provided.
The server displays health, stats, info at `/json`. The server displays health, stats, info at `/json`.
Use `--log-third-party-test-failures` along with `--debug` when you want extra info on the third-party proxy tests. This
can get very noisy if you have lots of third-party proxies so it's hidden behind an extra flag.
``` ```
=== Proxy Load Balancer === === Proxy Load Balancer ===
Usage of ./proxy-loadbalancer: Usage of /tmp/go-build1714785557/b001/exe/proxy-loadbalancer:
--config [string] --config [string]
Path to the config file Path to the config file
-d, --debug -d, --debug
Enable debug mode Enable debug mode
--v Print version and exit -l, --log-third-party-test-failures
-h, --help Print this help message Log third-party test debug info
-v Print version and exit
``` ```
## Special Headers ## Special Headers
The load balancer accepts special headers to control its behavior: The load balancer accepts special headers to control its behavior:
- `Thirdparty-Bypass`: don't use any third-party endpoints for this request. - `Thirdparty-Bypass`: don't use any third-party endpoints for this request.
- `Thirdparty-Include-Broken`: use all online endpoints for this request, including third-party ones that failed the special test. - `Thirdparty-Include-Broken`: use all online endpoints for this request, including third-party ones that failed the
special test.

View File

@ -1,17 +1,25 @@
# Port to run on. # Port to run on.
http_port: 9000 http_port: 9000
# How many proxies will be checked at once? # How many proxies will be checked at once?
proxy_checkers: 50 proxy_checkers: 50
# The interval between proxy checks in seconds. # The interval between proxy checks in seconds.
proxy_check_interval: 60 proxy_check_interval: 60
# URL to get a proxy's IP. # URL to get a proxy's IP.
ip_checker_url: https://api.ipify.org ip_checker_url: https://api.ipify.org
# Connection timeout for the proxies in seconds. # Connection timeout for the proxies in seconds.
proxy_connect_timeout: 60 proxy_connect_timeout: 60
# How many times to retry a proxy connection.
# On each retry a new proxy will be chosen.
proxy_connect_retries: 3
# Use `curl-impersonate` to pretend to be Chrome when testing proxies.
proxy_check_impersonate_chrome: false
proxy_check_impersonate_chrome_binary: ./curl_chrome116
# Your proxies. # Your proxies.
proxy_pool_ours: proxy_pool_ours:
@ -34,7 +42,7 @@ thirdparty_bypass_domains:
# Shuffle the proxy lists whenever the background thread refreshes them. # Shuffle the proxy lists whenever the background thread refreshes them.
# If false, round-robin on default order. # If false, round-robin on default order.
shuffle_proxies: false shuffle_proxies: false
# Don't allow requests to these domains through the proxy. # Don't allow requests to these domains through the proxy.
blocked_domains: blocked_domains:
@ -43,4 +51,4 @@ blocked_domains:
# Resolve specific domains through specific proxies. # Resolve specific domains through specific proxies.
# Proxies here are not validated. # Proxies here are not validated.
resolve_through: resolve_through:
github.com: http://1.2.3.4:3128 github.com: http://1.2.3.4:3128

View File

@ -1,7 +0,0 @@
proxy.py @ git+https://github.com/abhinavsingh/proxy.py.git@develop
redis
requests
coloredlogs
psutil
flask
async_timeout

View File

@ -2,7 +2,10 @@ package config
import ( import (
"errors" "errors"
"fmt"
"github.com/spf13/viper" "github.com/spf13/viper"
"os"
"os/exec"
"time" "time"
) )
@ -10,18 +13,21 @@ import (
var cfg *Config var cfg *Config
type Config struct { type Config struct {
HTTPPort string HTTPPort string
IpCheckerURL string IpCheckerURL string
MaxProxyCheckers int MaxProxyCheckers int
ProxyConnectTimeout time.Duration ProxyConnectTimeout time.Duration
ProxyPoolOurs []string ProxyPoolOurs []string
ProxyPoolThirdparty []string ProxyPoolThirdparty []string
ThirdpartyTestUrls []string ThirdpartyTestUrls []string
ThirdpartyBypassDomains []string ThirdpartyBypassDomains []string
ShuffleProxies bool ShuffleProxies bool
BlockedDomains []string BlockedDomains []string
ResolveThrough map[string]string ResolveThrough map[string]string
ProxyCheckInterval int ProxyCheckInterval int
ProxyCheckImpersonateChrome bool
ProxyCheckImpersonateChromeBinary string
ProxyConnectRetries int
} }
func SetConfig(configFile string) (*Config, error) { func SetConfig(configFile string) (*Config, error) {
@ -42,6 +48,9 @@ func SetConfig(configFile string) (*Config, error) {
viper.SetDefault("blocked_domains", make([]string, 0)) viper.SetDefault("blocked_domains", make([]string, 0))
viper.SetDefault("resolve_through", make(map[string]string)) viper.SetDefault("resolve_through", make(map[string]string))
viper.SetDefault("proxy_check_interval", 60) viper.SetDefault("proxy_check_interval", 60)
viper.SetDefault("proxy_check_impersonate_chrome", false)
viper.SetDefault("proxy_check_impersonate_chrome_binary", nil)
viper.SetDefault("proxy_connect_retries", 3)
err := viper.ReadInConfig() err := viper.ReadInConfig()
if err != nil { if err != nil {
@ -49,17 +58,20 @@ func SetConfig(configFile string) (*Config, error) {
} }
config := &Config{ config := &Config{
HTTPPort: viper.GetString("http_port"), HTTPPort: viper.GetString("http_port"),
IpCheckerURL: viper.GetString("ip_checker_url"), IpCheckerURL: viper.GetString("ip_checker_url"),
MaxProxyCheckers: viper.GetInt("proxy_checkers"), MaxProxyCheckers: viper.GetInt("proxy_checkers"),
ProxyPoolOurs: viper.GetStringSlice("proxy_pool_ours"), ProxyPoolOurs: viper.GetStringSlice("proxy_pool_ours"),
ProxyPoolThirdparty: viper.GetStringSlice("proxy_pool_thirdparty"), ProxyPoolThirdparty: viper.GetStringSlice("proxy_pool_thirdparty"),
ThirdpartyTestUrls: viper.GetStringSlice("thirdparty_test_urls"), ThirdpartyTestUrls: viper.GetStringSlice("thirdparty_test_urls"),
ThirdpartyBypassDomains: viper.GetStringSlice("thirdparty_bypass_domains"), ThirdpartyBypassDomains: viper.GetStringSlice("thirdparty_bypass_domains"),
ShuffleProxies: viper.GetBool("shuffle_proxies"), ShuffleProxies: viper.GetBool("shuffle_proxies"),
BlockedDomains: viper.GetStringSlice("blocked_domains"), BlockedDomains: viper.GetStringSlice("blocked_domains"),
ResolveThrough: viper.GetStringMapString("resolve_through"), ResolveThrough: viper.GetStringMapString("resolve_through"),
ProxyCheckInterval: viper.GetInt("proxy_check_interval"), ProxyCheckInterval: viper.GetInt("proxy_check_interval"),
ProxyCheckImpersonateChrome: viper.GetBool("proxy_check_impersonate_chrome"),
ProxyCheckImpersonateChromeBinary: viper.GetString("proxy_check_impersonate_chrome_binary"),
ProxyConnectRetries: viper.GetInt("proxy_connect_retries"),
} }
if len(config.ProxyPoolOurs) == 0 && len(config.ProxyPoolThirdparty) == 0 { if len(config.ProxyPoolOurs) == 0 && len(config.ProxyPoolThirdparty) == 0 {
@ -86,6 +98,21 @@ func SetConfig(configFile string) (*Config, error) {
return nil, proxyPoolThirdpartyErr return nil, proxyPoolThirdpartyErr
} }
if config.ProxyCheckImpersonateChrome {
if _, err := os.Stat(config.ProxyCheckImpersonateChromeBinary); os.IsNotExist(err) {
return nil, errors.New(fmt.Sprintf(`curl-impersonate-chrome binary does not exist: "%s"`, config.ProxyCheckImpersonateChromeBinary))
}
cmd := exec.Command(config.ProxyCheckImpersonateChromeBinary, "--help")
err = cmd.Run()
if err != nil {
return nil, errors.New(fmt.Sprintf(`curl-impersonate-chrome binary failed to run: %s`, err))
}
}
if config.ProxyConnectRetries <= 0 {
return nil, errors.New("proxy_connect_retries must be greater than 0")
}
cfg = config cfg = config
return config, nil return config, nil
} }

27
src/config/flags.go Normal file
View File

@ -0,0 +1,27 @@
package config
import "flag"
var CliArgs *CliConfig
type CliConfig struct {
ConfigFile string
Debug bool
Help bool
Version bool
LogThirdPartyTest bool
}
func ParseArgs() {
if CliArgs != nil {
panic("already defined")
}
CliArgs = &CliConfig{}
flag.StringVar(&CliArgs.ConfigFile, "config", "", "Path to the config file")
flag.BoolVar(&CliArgs.Debug, "d", false, "Enable debug mode")
flag.BoolVar(&CliArgs.Debug, "debug", false, "Enable debug mode")
flag.BoolVar(&CliArgs.Debug, "l", false, "Log third-party test debug info")
flag.BoolVar(&CliArgs.Debug, "log-third-party-test-failures", false, "Log third-party test debug info")
flag.BoolVar(&CliArgs.Version, "v", false, "Print version and exit")
flag.Parse()
}

View File

@ -13,26 +13,17 @@ import (
"runtime/debug" "runtime/debug"
) )
type cliConfig struct {
configFile string
initialCrawl bool
debug bool
disableElasticSync bool
help bool
version bool
}
var Version = "development" var Version = "development"
var VersionDate = "not set" var VersionDate = "not set"
func main() { func main() {
fmt.Println("=== Proxy Load Balancer ===") fmt.Println("=== Proxy Load Balancer ===")
cliArgs := parseArgs() config.ParseArgs()
if cliArgs.help { if config.CliArgs.Help {
flag.Usage() flag.Usage()
os.Exit(0) os.Exit(0)
} }
if cliArgs.version { if config.CliArgs.Version {
buildInfo, ok := debug.ReadBuildInfo() buildInfo, ok := debug.ReadBuildInfo()
if ok { if ok {
@ -51,7 +42,7 @@ func main() {
os.Exit(0) os.Exit(0)
} }
if cliArgs.debug { if config.CliArgs.Debug {
logging.InitLogger(logrus.DebugLevel) logging.InitLogger(logrus.DebugLevel)
} else { } else {
logging.InitLogger(logrus.InfoLevel) logging.InitLogger(logrus.InfoLevel)
@ -59,7 +50,7 @@ func main() {
log := logging.GetLogger() log := logging.GetLogger()
log.Debugln("Initializing...") log.Debugln("Initializing...")
if cliArgs.configFile == "" { if config.CliArgs.ConfigFile == "" {
exePath, err := os.Executable() exePath, err := os.Executable()
if err != nil { if err != nil {
panic(err) panic(err)
@ -70,20 +61,24 @@ func main() {
if _, err := os.Stat(filepath.Join(exeDir, "config.yaml")); err == nil { if _, err := os.Stat(filepath.Join(exeDir, "config.yaml")); err == nil {
log.Fatalln("Both config.yml and config.yaml exist in the executable directory. Please specify one with the --config flag.") log.Fatalln("Both config.yml and config.yaml exist in the executable directory. Please specify one with the --config flag.")
} }
cliArgs.configFile = filepath.Join(exeDir, "config.yml") config.CliArgs.ConfigFile = filepath.Join(exeDir, "config.yml")
} else if _, err := os.Stat(filepath.Join(exeDir, "config.yaml")); err == nil { } else if _, err := os.Stat(filepath.Join(exeDir, "config.yaml")); err == nil {
cliArgs.configFile = filepath.Join(exeDir, "config.yaml") config.CliArgs.ConfigFile = filepath.Join(exeDir, "config.yaml")
} else { } else {
log.Fatalln("No config file found in the executable directory. Please provide one with the --config flag.") log.Fatalln("No config file found in the executable directory. Please provide one with the --config flag.")
} }
} }
configData, err := config.SetConfig(cliArgs.configFile) configData, err := config.SetConfig(config.CliArgs.ConfigFile)
if err != nil { if err != nil {
log.Fatalf(`Failed to load config: %s`, err) log.Fatalf(`Failed to load config: %s`, err)
} }
log.Debugf(`Proxy check interval: %d sec`, config.GetConfig().ProxyCheckInterval) log.Debugf(`Proxy check interval: %d sec`, config.GetConfig().ProxyCheckInterval)
if config.GetConfig().ProxyCheckImpersonateChrome {
log.Debugf(`Using curl-impersonate binary: %s`, config.GetConfig().ProxyCheckImpersonateChromeBinary)
}
proxyCluster := proxy.NewForwardProxyCluster() proxyCluster := proxy.NewForwardProxyCluster()
go func() { go func() {
log.Fatal(http.ListenAndServe(":"+configData.HTTPPort, proxyCluster)) log.Fatal(http.ListenAndServe(":"+configData.HTTPPort, proxyCluster))
@ -96,13 +91,3 @@ func main() {
select {} select {}
} }
func parseArgs() cliConfig {
var cliArgs cliConfig
flag.StringVar(&cliArgs.configFile, "config", "", "Path to the config file")
flag.BoolVar(&cliArgs.debug, "d", false, "Enable debug mode")
flag.BoolVar(&cliArgs.debug, "debug", false, "Enable debug mode")
flag.BoolVar(&cliArgs.version, "v", false, "Print version and exit")
flag.Parse()
return cliArgs
}

View File

@ -1,11 +1,17 @@
package proxy package proxy
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"main/config" "main/config"
"net/http" "net/http"
"net/url" "net/url"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
) )
func sendRequestThroughProxy(pxy string, targetURL string) (string, error) { func sendRequestThroughProxy(pxy string, targetURL string) (string, error) {
@ -17,43 +23,128 @@ func sendRequestThroughProxy(pxy string, targetURL string) (string, error) {
if proxyUser != "" && proxyPass != "" { if proxyUser != "" && proxyPass != "" {
parsedProxyUrl.User = url.UserPassword(proxyUser, proxyPass) parsedProxyUrl.User = url.UserPassword(proxyUser, proxyPass)
} }
transport := &http.Transport{
Proxy: http.ProxyURL(parsedProxyUrl),
}
client := &http.Client{
Transport: transport,
Timeout: config.GetConfig().ProxyConnectTimeout,
}
req, err := http.NewRequest("GET", targetURL, nil) if !config.GetConfig().ProxyCheckImpersonateChrome {
if err != nil { transport := &http.Transport{
return "", err Proxy: http.ProxyURL(parsedProxyUrl),
} }
client := &http.Client{
Transport: transport,
Timeout: config.GetConfig().ProxyConnectTimeout,
}
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") req, err := http.NewRequest("GET", targetURL, nil)
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
req.Header.Set("Priority", "u=0, i")
req.Header.Set("Sec-Ch-Ua", `"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"`)
req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
req.Header.Set("Sec-Ch-Ua-Platform", `"Windows"`)
req.Header.Set("Sec-Fetch-Dest", "document")
req.Header.Set("Sec-Fetch-Mode", "navigate")
req.Header.Set("Sec-Fetch-Site", "cross-site")
req.Header.Set("Sec-Fetch-User", "?1")
req.Header.Set("Upgrade-Insecure-Requests", "1")
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return "", err return "", err
} }
return string(bodyBytes), nil
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
req.Header.Set("Priority", "u=0, i")
req.Header.Set("Sec-Ch-Ua", `Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"`)
req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
req.Header.Set("Sec-Ch-Ua-Platform", `"Windows"`)
req.Header.Set("Sec-Fetch-Dest", "document")
req.Header.Set("Sec-Fetch-Mode", "navigate")
req.Header.Set("Sec-Fetch-Site", "none")
req.Header.Set("Sec-Fetch-User", "?1")
req.Header.Set("Upgrade-Insecure-Requests", "1")
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36")
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
return string(bodyBytes), nil
}
return "", fmt.Errorf("bad response code %d", resp.StatusCode)
} else {
tmpfile, err := os.CreateTemp("", "response")
if err != nil {
log.Fatal(err)
}
defer os.Remove(tmpfile.Name()) // clean up
cmd := exec.Command(
config.GetConfig().ProxyCheckImpersonateChromeBinary,
"--proxy",
parsedProxyUrl.String(),
"-o",
tmpfile.Name(),
"-w",
"%{http_code}",
"--ciphers",
"TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA",
"-H",
`sec-ch-ua: "Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"`,
"-H",
`sec-ch-ua-mobile: ?0`,
"-H",
`sec-ch-ua-platform: "Windows"`,
"-H",
`Upgrade-Insecure-Requests: 1`,
"-H",
`User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36`,
"-H",
`Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7`,
"-H",
`Sec-Fetch-Site: none`,
"-H",
`Sec-Fetch-Mode: navigate`,
"-H",
`Sec-Fetch-User: ?1`,
"-H",
`Sec-Fetch-Dest: document`,
"-H",
`Accept-Encoding: gzip, deflate, br`,
"-H",
`Accept-Language: en-US,en;q=0.9`,
"--http2",
"--http2-no-server-push",
"--compressed",
"--tlsv1.2",
"--alps",
"--tls-permute-extensions",
"--cert-compression",
"brotli",
targetURL,
)
output, err := cmd.Output()
if err != nil {
var exitError *exec.ExitError
if errors.As(err, &exitError) {
return "", fmt.Errorf("command exited with code %d: %s", exitError.ExitCode(), exitError.Stderr)
}
return "", err
}
outputStr := strings.TrimSpace(string(output))
match, _ := regexp.MatchString(`^\d{3}$`, outputStr)
if !match {
return "", fmt.Errorf("unexpected output from curl command: %s", outputStr)
}
statusCode, err := strconv.Atoi(outputStr)
if err != nil {
return "", err
}
if statusCode != http.StatusOK {
return "", fmt.Errorf("bad response code %d", statusCode)
}
body, err := os.ReadFile(tmpfile.Name())
if err != nil {
return "", err
}
return string(body), nil
} }
return "", fmt.Errorf("bad response code %d", resp.StatusCode)
} }

View File

@ -19,8 +19,13 @@ var (
HeaderThirdpartyBypass = "Thirdparty-Bypass" HeaderThirdpartyBypass = "Thirdparty-Bypass"
) )
func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time) { func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time, errorMsg *string) {
log.Infof(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds()) msg := fmt.Sprintf(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds())
if *errorMsg == "" {
log.Infof(msg)
} else {
log.Errorf(`%s -- %s`, msg, *errorMsg)
}
} }
func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter, req *http.Request) (string, string, string, string, *url.URL, error) { func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter, req *http.Request) (string, string, string, string, *url.URL, error) {
@ -46,11 +51,11 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
return "", "", "", "", nil, errors.New(errStr) return "", "", "", "", nil, errors.New(errStr)
} }
headerIncludeBrokenThirdparty := req.Header.Get(HeaderThirdpartyIncludeBroken) headerIncludeBrokenThirdparty := req.Header.Get(HeaderThirdpartyIncludeBroken) != ""
req.Header.Del(HeaderThirdpartyIncludeBroken) req.Header.Del(HeaderThirdpartyIncludeBroken)
headerBypassThirdparty := req.Header.Get(HeaderThirdpartyBypass) headerBypassThirdparty := req.Header.Get(HeaderThirdpartyBypass) != ""
req.Header.Del(HeaderThirdpartyBypass) req.Header.Del(HeaderThirdpartyBypass)
if headerBypassThirdparty != "" && headerIncludeBrokenThirdparty != "" { if headerBypassThirdparty && headerIncludeBrokenThirdparty {
errStr := "duplicate options headers detected, rejecting request" errStr := "duplicate options headers detected, rejecting request"
http.Error(w, errStr, http.StatusBadRequest) http.Error(w, errStr, http.StatusBadRequest)
return "", "", "", "", nil, errors.New(errStr) return "", "", "", "", nil, errors.New(errStr)
@ -64,9 +69,9 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
if slices.Contains(config.GetConfig().ThirdpartyBypassDomains, urlHostname) { if slices.Contains(config.GetConfig().ThirdpartyBypassDomains, urlHostname) {
selectedProxy = p.getProxyFromOurs() selectedProxy = p.getProxyFromOurs()
} else { } else {
if headerIncludeBrokenThirdparty != "" { if headerIncludeBrokenThirdparty {
selectedProxy = p.getProxyFromAllWithBroken() selectedProxy = p.getProxyFromAllWithBroken()
} else if headerBypassThirdparty != "" { } else if headerBypassThirdparty {
selectedProxy = p.getProxyFromOurs() selectedProxy = p.getProxyFromOurs()
} else { } else {
selectedProxy = p.getProxyFromAll() selectedProxy = p.getProxyFromAll()
@ -85,7 +90,6 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
} }
return selectedProxy, proxyUser, proxyPass, proxyHost, parsedProxyUrl, nil return selectedProxy, proxyUser, proxyPass, proxyHost, parsedProxyUrl, nil
} }
func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.Request) { func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.Request) {
@ -93,17 +97,20 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr) remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
_, proxyUser, proxyPass, proxyHost, parsedProxyUrl, err := p.validateRequestAndGetProxy(w, req) _, proxyUser, proxyPass, proxyHost, parsedProxyUrl, err := p.validateRequestAndGetProxy(w, req)
if err != nil { if err != nil {
// Error has already been handled, just log and return.
if proxyHost == "" { if proxyHost == "" {
proxyHost = "none" proxyHost = "none"
} }
log.Debugf(`%s -> %s -> %s -- HTTP -- Rejecting request: %s`, remoteAddr, proxyHost, req.Host, err) log.Debugf(`%s -> %s -> %s -- HTTP -- Rejecting request: %s`, remoteAddr, proxyHost, req.Host, err)
return return
} }
var returnCode *int var returnCode *int
returnCode = new(int) returnCode = new(int)
*returnCode = -1 *returnCode = -1
defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime) var errorMsg *string
errorMsg = new(string)
*errorMsg = ""
defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime, errorMsg)
parsedProxyUrl.Scheme = "http" parsedProxyUrl.Scheme = "http"
if proxyUser != "" && proxyPass != "" { if proxyUser != "" && proxyPass != "" {
@ -118,7 +125,7 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
proxyReq, err := http.NewRequest(req.Method, req.URL.String(), req.Body) proxyReq, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
if err != nil { if err != nil {
log.Errorf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err) *errorMsg = fmt.Sprintf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err)
http.Error(w, "failed to make request to downstream", http.StatusInternalServerError) http.Error(w, "failed to make request to downstream", http.StatusInternalServerError)
return return
} }
@ -126,18 +133,25 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
copyHeader(proxyReq.Header, req.Header) copyHeader(proxyReq.Header, req.Header)
proxyReq.Header.Set("X-Forwarded-For", req.RemoteAddr) proxyReq.Header.Set("X-Forwarded-For", req.RemoteAddr)
resp, err := client.Do(proxyReq) for i := 0; i < config.GetConfig().ProxyConnectRetries; i++ { // Retry mechanic
if err != nil { resp, err := client.Do(proxyReq)
log.Errorf(`Failed to execute %s request to "%s": %s`, req.Method, req.URL.String(), err) if err != nil {
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable) *errorMsg = fmt.Sprintf(`Failed to execute %s request to "%s" - attempt %d/%d - %s`, req.Method, req.URL.String(), i+1, config.GetConfig().ProxyConnectRetries, err)
return if i < config.GetConfig().ProxyConnectRetries-1 {
continue
} else {
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
return
}
} else {
defer resp.Body.Close()
*returnCode = resp.StatusCode
copyHeader(w.Header(), resp.Header)
w.WriteHeader(resp.StatusCode)
io.Copy(w, resp.Body)
break
}
} }
defer resp.Body.Close()
*returnCode = resp.StatusCode
copyHeader(w.Header(), resp.Header)
w.WriteHeader(resp.StatusCode)
io.Copy(w, resp.Body)
} }
func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http.Request) { func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http.Request) {
@ -146,7 +160,6 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
targetHost, _, _ := net.SplitHostPort(req.Host) targetHost, _, _ := net.SplitHostPort(req.Host)
_, proxyUser, proxyPass, proxyHost, _, err := p.validateRequestAndGetProxy(w, req) _, proxyUser, proxyPass, proxyHost, _, err := p.validateRequestAndGetProxy(w, req)
if err != nil { if err != nil {
// Error has already been handled, just log and return.
if proxyHost == "" { if proxyHost == "" {
proxyHost = "none" proxyHost = "none"
} }
@ -156,14 +169,26 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
var returnCode *int var returnCode *int
returnCode = new(int) returnCode = new(int)
*returnCode = -1 *returnCode = -1
defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime) var errorMsg *string
errorMsg = new(string)
*errorMsg = ""
defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime, errorMsg)
// Start a connection to the downstream proxy server. var proxyConn net.Conn
proxyConn, err := net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout) for i := 0; i < config.GetConfig().ProxyConnectRetries; i++ {
if err != nil { // Start a connection to the downstream proxy server.
log.Errorf(`Failed to dial proxy %s - %s`, proxyHost, err) proxyConn, err = net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout)
http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable) if err != nil {
return *errorMsg = fmt.Sprintf(`Failed to dial proxy %s - attempt %d/%d - %s`, proxyHost, i+1, config.GetConfig().ProxyConnectRetries, err)
if i < config.GetConfig().ProxyConnectRetries-1 {
continue
} else {
http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable)
return
}
} else {
break
}
} }
// Proxy authentication // Proxy authentication
@ -178,7 +203,7 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
} }
resp, err := http.ReadResponse(bufio.NewReader(proxyConn), req) resp, err := http.ReadResponse(bufio.NewReader(proxyConn), req)
if resp == nil { if resp == nil {
log.Errorf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err) *errorMsg = fmt.Sprintf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable) http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
return return
} else if err != nil { } else if err != nil {
@ -189,14 +214,14 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
hj, ok := w.(http.Hijacker) hj, ok := w.(http.Hijacker)
if !ok { if !ok {
log.Errorf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost) *errorMsg = fmt.Sprintf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
http.Error(w, "failed to forward connection to downstream", http.StatusServiceUnavailable) http.Error(w, "failed to forward connection to downstream", http.StatusServiceUnavailable)
return return
} }
clientConn, _, err := hj.Hijack() clientConn, _, err := hj.Hijack()
if err != nil { if err != nil {
log.Errorf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost) *errorMsg = fmt.Sprintf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
http.Error(w, "failed to execute connection forwarding to downstream", http.StatusServiceUnavailable) http.Error(w, "failed to execute connection forwarding to downstream", http.StatusServiceUnavailable)
return return
} }

View File

@ -10,6 +10,8 @@ import (
"time" "time"
) )
// TODO: fix 503 errors returned during proxy checking process
func (p *ForwardProxyCluster) ValidateProxiesThread() { func (p *ForwardProxyCluster) ValidateProxiesThread() {
log.Infoln("Doing initial backend check, please wait...") log.Infoln("Doing initial backend check, please wait...")
started := false started := false
@ -17,6 +19,7 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
ctx := context.TODO() ctx := context.TODO()
for { for {
startTime := time.Now()
p.refreshInProgress = true p.refreshInProgress = true
allProxies := removeDuplicates(append(config.GetConfig().ProxyPoolOurs, config.GetConfig().ProxyPoolThirdparty...)) allProxies := removeDuplicates(append(config.GetConfig().ProxyPoolOurs, config.GetConfig().ProxyPoolThirdparty...))
newOurOnlineProxies := make([]string, 0) newOurOnlineProxies := make([]string, 0)
@ -57,10 +60,13 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
// Test the proxy. // Test the proxy.
ipAddr, testErr := sendRequestThroughProxy(pxy, config.GetConfig().IpCheckerURL) ipAddr, testErr := sendRequestThroughProxy(pxy, config.GetConfig().IpCheckerURL)
if testErr != nil { if testErr != nil {
log.Warnf("Validate - %s failed: %s", proxyHost, testErr)
if isThirdparty(pxy) { if isThirdparty(pxy) {
if config.CliArgs.LogThirdPartyTest {
log.Debugf("Validate - %s failed: %s", proxyHost, testErr)
}
newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy) newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
} else { } else {
log.Debugf("Validate - %s failed: %s", proxyHost, testErr)
newOurOfflineProxies = append(newOurOfflineProxies, pxy) newOurOfflineProxies = append(newOurOfflineProxies, pxy)
} }
return return
@ -84,7 +90,9 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
if bv3hiErr != nil { if bv3hiErr != nil {
okToAdd = false okToAdd = false
newThirdpartyBrokenProxies = append(newThirdpartyBrokenProxies, pxy) newThirdpartyBrokenProxies = append(newThirdpartyBrokenProxies, pxy)
log.Debugf("Validate - %s failed third-party test: %s", proxyHost, bv3hiErr) if config.CliArgs.LogThirdPartyTest {
log.Debugf(`%s failed third-party test for URL "%s" -- %s`, proxyHost, d, bv3hiErr)
}
break break
} }
} }
@ -121,8 +129,8 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
} }
p.mu.RLock() p.mu.RLock()
log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d", log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d, Elapsed: %s",
len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+len(p.thirdpartyOnlineProxies), len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+len(p.thirdpartyOnlineProxies), time.Since(startTime),
) )
p.mu.RUnlock() p.mu.RUnlock()