Compare commits
8 Commits
Author | SHA1 | Date |
---|---|---|
|
d6c916d019 | |
|
88b0a62272 | |
|
beb78ce292 | |
|
3f27ceff89 | |
|
bffd709c41 | |
|
efefc76929 | |
|
e93f353860 | |
|
c3e60eaca9 |
47
README.md
47
README.md
|
@ -2,41 +2,58 @@
|
|||
|
||||
_A round-robin load balancer for HTTP proxies._
|
||||
|
||||
This is a simple proxy load balancer that will route requests to a cluster of proxy backends in a round-robin fashion. This makes it easy to connect your clients to a large number of proxy servers without worrying about implementing anything special clientside.
|
||||
This is a simple proxy load balancer that will route requests to a cluster of proxy backends in a round-robin fashion.
|
||||
This makes it easy to connect your clients to a large number of proxy servers without worrying about implementing
|
||||
anything special client-side.
|
||||
|
||||
This proxy server will transparently forward HTTPS requests without terminating them, meaning a self-signed certificate is not required. Downstream HTTPS proxy servers are not supported.
|
||||
This proxy server will transparently forward HTTPS requests without terminating them, meaning a self-signed certificate
|
||||
is not required. Downstream HTTPS proxy servers are not supported.
|
||||
|
||||
Memory usage sits at around 25M under load.
|
||||
|
||||
## Install
|
||||
|
||||
1. Download the latest release from [/releases](https://git.evulid.cc/cyberes/proxy-loadbalancer/releases) or run `./build.sh` to build the program locally.
|
||||
2. `cp config.example.yml config.yml`
|
||||
3. Edit the config.
|
||||
4. Start the loadbalancer with `./proxy-loadbalancer --config [path to your config.yml]`
|
||||
1. Download the latest release from [/releases](https://git.evulid.cc/cyberes/proxy-loadbalancer/releases) or
|
||||
run `./build.sh` to build the program locally.
|
||||
2. `cp config.example.yml config.yml`
|
||||
3. Edit the config.
|
||||
4. Start the loadbalancer with `./proxy-loadbalancer --config [path to your config.yml]`
|
||||
|
||||
The load balancer has experimental support for using [curl-impersonate](https://github.com/lwthiker/curl-impersonate) to masquerade as the Chrome browser when
|
||||
performing proxy checks.
|
||||
|
||||
1. Download `*.x86_64-linux-gnu.tar.gz ` from https://github.com/lwthiker/curl-impersonate/releases
|
||||
2. Set `proxy_check_impersonate_chrome: true`
|
||||
3. Enter the path to the `curl_chrome116` binary in `proxy_check_impersonate_chrome_binary`
|
||||
|
||||
## Use
|
||||
|
||||
You can run your own "public IP delivery server" `canihazip` <https://git.evulid.cc/cyberes/canihazip> or use the default `api.ipify.org`
|
||||
You can run your own "public IP delivery server" `canihazip` <https://git.evulid.cc/cyberes/canihazip> or use the
|
||||
default `api.ipify.org`
|
||||
|
||||
An example systemd service `loadbalancer.service` is provided.
|
||||
|
||||
The server displays health, stats, info at `/json`.
|
||||
|
||||
Use `--log-third-party-test-failures` along with `--debug` when you want extra info on the third-party proxy tests. This
|
||||
can get very noisy if you have lots of third-party proxies so it's hidden behind an extra flag.
|
||||
|
||||
```
|
||||
=== Proxy Load Balancer ===
|
||||
Usage of ./proxy-loadbalancer:
|
||||
Usage of /tmp/go-build1714785557/b001/exe/proxy-loadbalancer:
|
||||
--config [string]
|
||||
Path to the config file
|
||||
-d, --debug
|
||||
Enable debug mode
|
||||
--v Print version and exit
|
||||
-h, --help Print this help message
|
||||
Path to the config file
|
||||
-d, --debug
|
||||
Enable debug mode
|
||||
-l, --log-third-party-test-failures
|
||||
Log third-party test debug info
|
||||
-v Print version and exit
|
||||
```
|
||||
|
||||
## Special Headers
|
||||
|
||||
The load balancer accepts special headers to control its behavior:
|
||||
|
||||
- `Thirdparty-Bypass`: don't use any third-party endpoints for this request.
|
||||
- `Thirdparty-Include-Broken`: use all online endpoints for this request, including third-party ones that failed the special test.
|
||||
- `Thirdparty-Bypass`: don't use any third-party endpoints for this request.
|
||||
- `Thirdparty-Include-Broken`: use all online endpoints for this request, including third-party ones that failed the
|
||||
special test.
|
||||
|
|
|
@ -1,17 +1,25 @@
|
|||
# Port to run on.
|
||||
http_port: 9000
|
||||
http_port: 9000
|
||||
|
||||
# How many proxies will be checked at once?
|
||||
proxy_checkers: 50
|
||||
proxy_checkers: 50
|
||||
|
||||
# The interval between proxy checks in seconds.
|
||||
proxy_check_interval: 60
|
||||
proxy_check_interval: 60
|
||||
|
||||
# URL to get a proxy's IP.
|
||||
ip_checker_url: https://api.ipify.org
|
||||
ip_checker_url: https://api.ipify.org
|
||||
|
||||
# Connection timeout for the proxies in seconds.
|
||||
proxy_connect_timeout: 60
|
||||
proxy_connect_timeout: 60
|
||||
|
||||
# How many times to retry a proxy connection.
|
||||
# On each retry a new proxy will be chosen.
|
||||
proxy_connect_retries: 3
|
||||
|
||||
# Use `curl-impersonate` to pretend to be Chrome when testing proxies.
|
||||
proxy_check_impersonate_chrome: false
|
||||
proxy_check_impersonate_chrome_binary: ./curl_chrome116
|
||||
|
||||
# Your proxies.
|
||||
proxy_pool_ours:
|
||||
|
@ -34,7 +42,7 @@ thirdparty_bypass_domains:
|
|||
|
||||
# Shuffle the proxy lists whenever the background thread refreshes them.
|
||||
# If false, round-robin on default order.
|
||||
shuffle_proxies: false
|
||||
shuffle_proxies: false
|
||||
|
||||
# Don't allow requests to these domains through the proxy.
|
||||
blocked_domains:
|
||||
|
@ -43,4 +51,4 @@ blocked_domains:
|
|||
# Resolve specific domains through specific proxies.
|
||||
# Proxies here are not validated.
|
||||
resolve_through:
|
||||
github.com: http://1.2.3.4:3128
|
||||
github.com: http://1.2.3.4:3128
|
||||
|
|
|
@ -1,7 +0,0 @@
|
|||
proxy.py @ git+https://github.com/abhinavsingh/proxy.py.git@develop
|
||||
redis
|
||||
requests
|
||||
coloredlogs
|
||||
psutil
|
||||
flask
|
||||
async_timeout
|
|
@ -2,7 +2,10 @@ package config
|
|||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/spf13/viper"
|
||||
"os"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
@ -10,18 +13,21 @@ import (
|
|||
var cfg *Config
|
||||
|
||||
type Config struct {
|
||||
HTTPPort string
|
||||
IpCheckerURL string
|
||||
MaxProxyCheckers int
|
||||
ProxyConnectTimeout time.Duration
|
||||
ProxyPoolOurs []string
|
||||
ProxyPoolThirdparty []string
|
||||
ThirdpartyTestUrls []string
|
||||
ThirdpartyBypassDomains []string
|
||||
ShuffleProxies bool
|
||||
BlockedDomains []string
|
||||
ResolveThrough map[string]string
|
||||
ProxyCheckInterval int
|
||||
HTTPPort string
|
||||
IpCheckerURL string
|
||||
MaxProxyCheckers int
|
||||
ProxyConnectTimeout time.Duration
|
||||
ProxyPoolOurs []string
|
||||
ProxyPoolThirdparty []string
|
||||
ThirdpartyTestUrls []string
|
||||
ThirdpartyBypassDomains []string
|
||||
ShuffleProxies bool
|
||||
BlockedDomains []string
|
||||
ResolveThrough map[string]string
|
||||
ProxyCheckInterval int
|
||||
ProxyCheckImpersonateChrome bool
|
||||
ProxyCheckImpersonateChromeBinary string
|
||||
ProxyConnectRetries int
|
||||
}
|
||||
|
||||
func SetConfig(configFile string) (*Config, error) {
|
||||
|
@ -42,6 +48,9 @@ func SetConfig(configFile string) (*Config, error) {
|
|||
viper.SetDefault("blocked_domains", make([]string, 0))
|
||||
viper.SetDefault("resolve_through", make(map[string]string))
|
||||
viper.SetDefault("proxy_check_interval", 60)
|
||||
viper.SetDefault("proxy_check_impersonate_chrome", false)
|
||||
viper.SetDefault("proxy_check_impersonate_chrome_binary", nil)
|
||||
viper.SetDefault("proxy_connect_retries", 3)
|
||||
|
||||
err := viper.ReadInConfig()
|
||||
if err != nil {
|
||||
|
@ -49,17 +58,20 @@ func SetConfig(configFile string) (*Config, error) {
|
|||
}
|
||||
|
||||
config := &Config{
|
||||
HTTPPort: viper.GetString("http_port"),
|
||||
IpCheckerURL: viper.GetString("ip_checker_url"),
|
||||
MaxProxyCheckers: viper.GetInt("proxy_checkers"),
|
||||
ProxyPoolOurs: viper.GetStringSlice("proxy_pool_ours"),
|
||||
ProxyPoolThirdparty: viper.GetStringSlice("proxy_pool_thirdparty"),
|
||||
ThirdpartyTestUrls: viper.GetStringSlice("thirdparty_test_urls"),
|
||||
ThirdpartyBypassDomains: viper.GetStringSlice("thirdparty_bypass_domains"),
|
||||
ShuffleProxies: viper.GetBool("shuffle_proxies"),
|
||||
BlockedDomains: viper.GetStringSlice("blocked_domains"),
|
||||
ResolveThrough: viper.GetStringMapString("resolve_through"),
|
||||
ProxyCheckInterval: viper.GetInt("proxy_check_interval"),
|
||||
HTTPPort: viper.GetString("http_port"),
|
||||
IpCheckerURL: viper.GetString("ip_checker_url"),
|
||||
MaxProxyCheckers: viper.GetInt("proxy_checkers"),
|
||||
ProxyPoolOurs: viper.GetStringSlice("proxy_pool_ours"),
|
||||
ProxyPoolThirdparty: viper.GetStringSlice("proxy_pool_thirdparty"),
|
||||
ThirdpartyTestUrls: viper.GetStringSlice("thirdparty_test_urls"),
|
||||
ThirdpartyBypassDomains: viper.GetStringSlice("thirdparty_bypass_domains"),
|
||||
ShuffleProxies: viper.GetBool("shuffle_proxies"),
|
||||
BlockedDomains: viper.GetStringSlice("blocked_domains"),
|
||||
ResolveThrough: viper.GetStringMapString("resolve_through"),
|
||||
ProxyCheckInterval: viper.GetInt("proxy_check_interval"),
|
||||
ProxyCheckImpersonateChrome: viper.GetBool("proxy_check_impersonate_chrome"),
|
||||
ProxyCheckImpersonateChromeBinary: viper.GetString("proxy_check_impersonate_chrome_binary"),
|
||||
ProxyConnectRetries: viper.GetInt("proxy_connect_retries"),
|
||||
}
|
||||
|
||||
if len(config.ProxyPoolOurs) == 0 && len(config.ProxyPoolThirdparty) == 0 {
|
||||
|
@ -86,6 +98,21 @@ func SetConfig(configFile string) (*Config, error) {
|
|||
return nil, proxyPoolThirdpartyErr
|
||||
}
|
||||
|
||||
if config.ProxyCheckImpersonateChrome {
|
||||
if _, err := os.Stat(config.ProxyCheckImpersonateChromeBinary); os.IsNotExist(err) {
|
||||
return nil, errors.New(fmt.Sprintf(`curl-impersonate-chrome binary does not exist: "%s"`, config.ProxyCheckImpersonateChromeBinary))
|
||||
}
|
||||
cmd := exec.Command(config.ProxyCheckImpersonateChromeBinary, "--help")
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
return nil, errors.New(fmt.Sprintf(`curl-impersonate-chrome binary failed to run: %s`, err))
|
||||
}
|
||||
}
|
||||
|
||||
if config.ProxyConnectRetries <= 0 {
|
||||
return nil, errors.New("proxy_connect_retries must be greater than 0")
|
||||
}
|
||||
|
||||
cfg = config
|
||||
return config, nil
|
||||
}
|
||||
|
|
|
@ -0,0 +1,27 @@
|
|||
package config
|
||||
|
||||
import "flag"
|
||||
|
||||
var CliArgs *CliConfig
|
||||
|
||||
type CliConfig struct {
|
||||
ConfigFile string
|
||||
Debug bool
|
||||
Help bool
|
||||
Version bool
|
||||
LogThirdPartyTest bool
|
||||
}
|
||||
|
||||
func ParseArgs() {
|
||||
if CliArgs != nil {
|
||||
panic("already defined")
|
||||
}
|
||||
CliArgs = &CliConfig{}
|
||||
flag.StringVar(&CliArgs.ConfigFile, "config", "", "Path to the config file")
|
||||
flag.BoolVar(&CliArgs.Debug, "d", false, "Enable debug mode")
|
||||
flag.BoolVar(&CliArgs.Debug, "debug", false, "Enable debug mode")
|
||||
flag.BoolVar(&CliArgs.Debug, "l", false, "Log third-party test debug info")
|
||||
flag.BoolVar(&CliArgs.Debug, "log-third-party-test-failures", false, "Log third-party test debug info")
|
||||
flag.BoolVar(&CliArgs.Version, "v", false, "Print version and exit")
|
||||
flag.Parse()
|
||||
}
|
|
@ -13,26 +13,17 @@ import (
|
|||
"runtime/debug"
|
||||
)
|
||||
|
||||
type cliConfig struct {
|
||||
configFile string
|
||||
initialCrawl bool
|
||||
debug bool
|
||||
disableElasticSync bool
|
||||
help bool
|
||||
version bool
|
||||
}
|
||||
|
||||
var Version = "development"
|
||||
var VersionDate = "not set"
|
||||
|
||||
func main() {
|
||||
fmt.Println("=== Proxy Load Balancer ===")
|
||||
cliArgs := parseArgs()
|
||||
if cliArgs.help {
|
||||
config.ParseArgs()
|
||||
if config.CliArgs.Help {
|
||||
flag.Usage()
|
||||
os.Exit(0)
|
||||
}
|
||||
if cliArgs.version {
|
||||
if config.CliArgs.Version {
|
||||
buildInfo, ok := debug.ReadBuildInfo()
|
||||
|
||||
if ok {
|
||||
|
@ -51,7 +42,7 @@ func main() {
|
|||
os.Exit(0)
|
||||
}
|
||||
|
||||
if cliArgs.debug {
|
||||
if config.CliArgs.Debug {
|
||||
logging.InitLogger(logrus.DebugLevel)
|
||||
} else {
|
||||
logging.InitLogger(logrus.InfoLevel)
|
||||
|
@ -59,7 +50,7 @@ func main() {
|
|||
log := logging.GetLogger()
|
||||
log.Debugln("Initializing...")
|
||||
|
||||
if cliArgs.configFile == "" {
|
||||
if config.CliArgs.ConfigFile == "" {
|
||||
exePath, err := os.Executable()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
@ -70,20 +61,24 @@ func main() {
|
|||
if _, err := os.Stat(filepath.Join(exeDir, "config.yaml")); err == nil {
|
||||
log.Fatalln("Both config.yml and config.yaml exist in the executable directory. Please specify one with the --config flag.")
|
||||
}
|
||||
cliArgs.configFile = filepath.Join(exeDir, "config.yml")
|
||||
config.CliArgs.ConfigFile = filepath.Join(exeDir, "config.yml")
|
||||
} else if _, err := os.Stat(filepath.Join(exeDir, "config.yaml")); err == nil {
|
||||
cliArgs.configFile = filepath.Join(exeDir, "config.yaml")
|
||||
config.CliArgs.ConfigFile = filepath.Join(exeDir, "config.yaml")
|
||||
} else {
|
||||
log.Fatalln("No config file found in the executable directory. Please provide one with the --config flag.")
|
||||
}
|
||||
}
|
||||
configData, err := config.SetConfig(cliArgs.configFile)
|
||||
configData, err := config.SetConfig(config.CliArgs.ConfigFile)
|
||||
if err != nil {
|
||||
log.Fatalf(`Failed to load config: %s`, err)
|
||||
}
|
||||
|
||||
log.Debugf(`Proxy check interval: %d sec`, config.GetConfig().ProxyCheckInterval)
|
||||
|
||||
if config.GetConfig().ProxyCheckImpersonateChrome {
|
||||
log.Debugf(`Using curl-impersonate binary: %s`, config.GetConfig().ProxyCheckImpersonateChromeBinary)
|
||||
}
|
||||
|
||||
proxyCluster := proxy.NewForwardProxyCluster()
|
||||
go func() {
|
||||
log.Fatal(http.ListenAndServe(":"+configData.HTTPPort, proxyCluster))
|
||||
|
@ -96,13 +91,3 @@ func main() {
|
|||
|
||||
select {}
|
||||
}
|
||||
|
||||
func parseArgs() cliConfig {
|
||||
var cliArgs cliConfig
|
||||
flag.StringVar(&cliArgs.configFile, "config", "", "Path to the config file")
|
||||
flag.BoolVar(&cliArgs.debug, "d", false, "Enable debug mode")
|
||||
flag.BoolVar(&cliArgs.debug, "debug", false, "Enable debug mode")
|
||||
flag.BoolVar(&cliArgs.version, "v", false, "Print version and exit")
|
||||
flag.Parse()
|
||||
return cliArgs
|
||||
}
|
||||
|
|
|
@ -1,11 +1,17 @@
|
|||
package proxy
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"main/config"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func sendRequestThroughProxy(pxy string, targetURL string) (string, error) {
|
||||
|
@ -17,43 +23,128 @@ func sendRequestThroughProxy(pxy string, targetURL string) (string, error) {
|
|||
if proxyUser != "" && proxyPass != "" {
|
||||
parsedProxyUrl.User = url.UserPassword(proxyUser, proxyPass)
|
||||
}
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyURL(parsedProxyUrl),
|
||||
}
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: config.GetConfig().ProxyConnectTimeout,
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", targetURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !config.GetConfig().ProxyCheckImpersonateChrome {
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyURL(parsedProxyUrl),
|
||||
}
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: config.GetConfig().ProxyConnectTimeout,
|
||||
}
|
||||
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
req.Header.Set("Priority", "u=0, i")
|
||||
req.Header.Set("Sec-Ch-Ua", `"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"`)
|
||||
req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
|
||||
req.Header.Set("Sec-Ch-Ua-Platform", `"Windows"`)
|
||||
req.Header.Set("Sec-Fetch-Dest", "document")
|
||||
req.Header.Set("Sec-Fetch-Mode", "navigate")
|
||||
req.Header.Set("Sec-Fetch-Site", "cross-site")
|
||||
req.Header.Set("Sec-Fetch-User", "?1")
|
||||
req.Header.Set("Upgrade-Insecure-Requests", "1")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
req, err := http.NewRequest("GET", targetURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(bodyBytes), nil
|
||||
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
req.Header.Set("Priority", "u=0, i")
|
||||
req.Header.Set("Sec-Ch-Ua", `Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"`)
|
||||
req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
|
||||
req.Header.Set("Sec-Ch-Ua-Platform", `"Windows"`)
|
||||
req.Header.Set("Sec-Fetch-Dest", "document")
|
||||
req.Header.Set("Sec-Fetch-Mode", "navigate")
|
||||
req.Header.Set("Sec-Fetch-Site", "none")
|
||||
req.Header.Set("Sec-Fetch-User", "?1")
|
||||
req.Header.Set("Upgrade-Insecure-Requests", "1")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(bodyBytes), nil
|
||||
}
|
||||
return "", fmt.Errorf("bad response code %d", resp.StatusCode)
|
||||
} else {
|
||||
tmpfile, err := os.CreateTemp("", "response")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer os.Remove(tmpfile.Name()) // clean up
|
||||
|
||||
cmd := exec.Command(
|
||||
config.GetConfig().ProxyCheckImpersonateChromeBinary,
|
||||
"--proxy",
|
||||
parsedProxyUrl.String(),
|
||||
"-o",
|
||||
tmpfile.Name(),
|
||||
"-w",
|
||||
"%{http_code}",
|
||||
"--ciphers",
|
||||
"TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA",
|
||||
"-H",
|
||||
`sec-ch-ua: "Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"`,
|
||||
"-H",
|
||||
`sec-ch-ua-mobile: ?0`,
|
||||
"-H",
|
||||
`sec-ch-ua-platform: "Windows"`,
|
||||
"-H",
|
||||
`Upgrade-Insecure-Requests: 1`,
|
||||
"-H",
|
||||
`User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36`,
|
||||
"-H",
|
||||
`Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7`,
|
||||
"-H",
|
||||
`Sec-Fetch-Site: none`,
|
||||
"-H",
|
||||
`Sec-Fetch-Mode: navigate`,
|
||||
"-H",
|
||||
`Sec-Fetch-User: ?1`,
|
||||
"-H",
|
||||
`Sec-Fetch-Dest: document`,
|
||||
"-H",
|
||||
`Accept-Encoding: gzip, deflate, br`,
|
||||
"-H",
|
||||
`Accept-Language: en-US,en;q=0.9`,
|
||||
"--http2",
|
||||
"--http2-no-server-push",
|
||||
"--compressed",
|
||||
"--tlsv1.2",
|
||||
"--alps",
|
||||
"--tls-permute-extensions",
|
||||
"--cert-compression",
|
||||
"brotli",
|
||||
targetURL,
|
||||
)
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
var exitError *exec.ExitError
|
||||
if errors.As(err, &exitError) {
|
||||
return "", fmt.Errorf("command exited with code %d: %s", exitError.ExitCode(), exitError.Stderr)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
outputStr := strings.TrimSpace(string(output))
|
||||
match, _ := regexp.MatchString(`^\d{3}$`, outputStr)
|
||||
if !match {
|
||||
return "", fmt.Errorf("unexpected output from curl command: %s", outputStr)
|
||||
}
|
||||
|
||||
statusCode, err := strconv.Atoi(outputStr)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if statusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("bad response code %d", statusCode)
|
||||
}
|
||||
|
||||
body, err := os.ReadFile(tmpfile.Name())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
||||
return "", fmt.Errorf("bad response code %d", resp.StatusCode)
|
||||
}
|
||||
|
|
|
@ -19,8 +19,13 @@ var (
|
|||
HeaderThirdpartyBypass = "Thirdparty-Bypass"
|
||||
)
|
||||
|
||||
func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time) {
|
||||
log.Infof(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds())
|
||||
func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time, errorMsg *string) {
|
||||
msg := fmt.Sprintf(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds())
|
||||
if *errorMsg == "" {
|
||||
log.Infof(msg)
|
||||
} else {
|
||||
log.Errorf(`%s -- %s`, msg, *errorMsg)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter, req *http.Request) (string, string, string, string, *url.URL, error) {
|
||||
|
@ -46,11 +51,11 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
|
|||
return "", "", "", "", nil, errors.New(errStr)
|
||||
}
|
||||
|
||||
headerIncludeBrokenThirdparty := req.Header.Get(HeaderThirdpartyIncludeBroken)
|
||||
headerIncludeBrokenThirdparty := req.Header.Get(HeaderThirdpartyIncludeBroken) != ""
|
||||
req.Header.Del(HeaderThirdpartyIncludeBroken)
|
||||
headerBypassThirdparty := req.Header.Get(HeaderThirdpartyBypass)
|
||||
headerBypassThirdparty := req.Header.Get(HeaderThirdpartyBypass) != ""
|
||||
req.Header.Del(HeaderThirdpartyBypass)
|
||||
if headerBypassThirdparty != "" && headerIncludeBrokenThirdparty != "" {
|
||||
if headerBypassThirdparty && headerIncludeBrokenThirdparty {
|
||||
errStr := "duplicate options headers detected, rejecting request"
|
||||
http.Error(w, errStr, http.StatusBadRequest)
|
||||
return "", "", "", "", nil, errors.New(errStr)
|
||||
|
@ -64,9 +69,9 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
|
|||
if slices.Contains(config.GetConfig().ThirdpartyBypassDomains, urlHostname) {
|
||||
selectedProxy = p.getProxyFromOurs()
|
||||
} else {
|
||||
if headerIncludeBrokenThirdparty != "" {
|
||||
if headerIncludeBrokenThirdparty {
|
||||
selectedProxy = p.getProxyFromAllWithBroken()
|
||||
} else if headerBypassThirdparty != "" {
|
||||
} else if headerBypassThirdparty {
|
||||
selectedProxy = p.getProxyFromOurs()
|
||||
} else {
|
||||
selectedProxy = p.getProxyFromAll()
|
||||
|
@ -85,7 +90,6 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
|
|||
}
|
||||
|
||||
return selectedProxy, proxyUser, proxyPass, proxyHost, parsedProxyUrl, nil
|
||||
|
||||
}
|
||||
|
||||
func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.Request) {
|
||||
|
@ -93,17 +97,20 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
|
|||
remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
|
||||
_, proxyUser, proxyPass, proxyHost, parsedProxyUrl, err := p.validateRequestAndGetProxy(w, req)
|
||||
if err != nil {
|
||||
// Error has already been handled, just log and return.
|
||||
if proxyHost == "" {
|
||||
proxyHost = "none"
|
||||
}
|
||||
log.Debugf(`%s -> %s -> %s -- HTTP -- Rejecting request: %s`, remoteAddr, proxyHost, req.Host, err)
|
||||
return
|
||||
}
|
||||
|
||||
var returnCode *int
|
||||
returnCode = new(int)
|
||||
*returnCode = -1
|
||||
defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime)
|
||||
var errorMsg *string
|
||||
errorMsg = new(string)
|
||||
*errorMsg = ""
|
||||
defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime, errorMsg)
|
||||
|
||||
parsedProxyUrl.Scheme = "http"
|
||||
if proxyUser != "" && proxyPass != "" {
|
||||
|
@ -118,7 +125,7 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
|
|||
|
||||
proxyReq, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
|
||||
if err != nil {
|
||||
log.Errorf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err)
|
||||
*errorMsg = fmt.Sprintf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err)
|
||||
http.Error(w, "failed to make request to downstream", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
@ -126,18 +133,25 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
|
|||
copyHeader(proxyReq.Header, req.Header)
|
||||
proxyReq.Header.Set("X-Forwarded-For", req.RemoteAddr)
|
||||
|
||||
resp, err := client.Do(proxyReq)
|
||||
if err != nil {
|
||||
log.Errorf(`Failed to execute %s request to "%s": %s`, req.Method, req.URL.String(), err)
|
||||
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
for i := 0; i < config.GetConfig().ProxyConnectRetries; i++ { // Retry mechanic
|
||||
resp, err := client.Do(proxyReq)
|
||||
if err != nil {
|
||||
*errorMsg = fmt.Sprintf(`Failed to execute %s request to "%s" - attempt %d/%d - %s`, req.Method, req.URL.String(), i+1, config.GetConfig().ProxyConnectRetries, err)
|
||||
if i < config.GetConfig().ProxyConnectRetries-1 {
|
||||
continue
|
||||
} else {
|
||||
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
defer resp.Body.Close()
|
||||
*returnCode = resp.StatusCode
|
||||
copyHeader(w.Header(), resp.Header)
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
io.Copy(w, resp.Body)
|
||||
break
|
||||
}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
*returnCode = resp.StatusCode
|
||||
|
||||
copyHeader(w.Header(), resp.Header)
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http.Request) {
|
||||
|
@ -146,7 +160,6 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
|
|||
targetHost, _, _ := net.SplitHostPort(req.Host)
|
||||
_, proxyUser, proxyPass, proxyHost, _, err := p.validateRequestAndGetProxy(w, req)
|
||||
if err != nil {
|
||||
// Error has already been handled, just log and return.
|
||||
if proxyHost == "" {
|
||||
proxyHost = "none"
|
||||
}
|
||||
|
@ -156,14 +169,26 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
|
|||
var returnCode *int
|
||||
returnCode = new(int)
|
||||
*returnCode = -1
|
||||
defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime)
|
||||
var errorMsg *string
|
||||
errorMsg = new(string)
|
||||
*errorMsg = ""
|
||||
defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime, errorMsg)
|
||||
|
||||
// Start a connection to the downstream proxy server.
|
||||
proxyConn, err := net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout)
|
||||
if err != nil {
|
||||
log.Errorf(`Failed to dial proxy %s - %s`, proxyHost, err)
|
||||
http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
var proxyConn net.Conn
|
||||
for i := 0; i < config.GetConfig().ProxyConnectRetries; i++ {
|
||||
// Start a connection to the downstream proxy server.
|
||||
proxyConn, err = net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout)
|
||||
if err != nil {
|
||||
*errorMsg = fmt.Sprintf(`Failed to dial proxy %s - attempt %d/%d - %s`, proxyHost, i+1, config.GetConfig().ProxyConnectRetries, err)
|
||||
if i < config.GetConfig().ProxyConnectRetries-1 {
|
||||
continue
|
||||
} else {
|
||||
http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Proxy authentication
|
||||
|
@ -178,7 +203,7 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
|
|||
}
|
||||
resp, err := http.ReadResponse(bufio.NewReader(proxyConn), req)
|
||||
if resp == nil {
|
||||
log.Errorf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
|
||||
*errorMsg = fmt.Sprintf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
|
||||
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
} else if err != nil {
|
||||
|
@ -189,14 +214,14 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
|
|||
w.WriteHeader(http.StatusOK)
|
||||
hj, ok := w.(http.Hijacker)
|
||||
if !ok {
|
||||
log.Errorf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
|
||||
*errorMsg = fmt.Sprintf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
|
||||
http.Error(w, "failed to forward connection to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
clientConn, _, err := hj.Hijack()
|
||||
if err != nil {
|
||||
log.Errorf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
|
||||
*errorMsg = fmt.Sprintf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
|
||||
http.Error(w, "failed to execute connection forwarding to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
|
|
@ -10,6 +10,8 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
// TODO: fix 503 errors returned during proxy checking process
|
||||
|
||||
func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
||||
log.Infoln("Doing initial backend check, please wait...")
|
||||
started := false
|
||||
|
@ -17,6 +19,7 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
|||
ctx := context.TODO()
|
||||
|
||||
for {
|
||||
startTime := time.Now()
|
||||
p.refreshInProgress = true
|
||||
allProxies := removeDuplicates(append(config.GetConfig().ProxyPoolOurs, config.GetConfig().ProxyPoolThirdparty...))
|
||||
newOurOnlineProxies := make([]string, 0)
|
||||
|
@ -57,10 +60,13 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
|||
// Test the proxy.
|
||||
ipAddr, testErr := sendRequestThroughProxy(pxy, config.GetConfig().IpCheckerURL)
|
||||
if testErr != nil {
|
||||
log.Warnf("Validate - %s failed: %s", proxyHost, testErr)
|
||||
if isThirdparty(pxy) {
|
||||
if config.CliArgs.LogThirdPartyTest {
|
||||
log.Debugf("Validate - %s failed: %s", proxyHost, testErr)
|
||||
}
|
||||
newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
|
||||
} else {
|
||||
log.Debugf("Validate - %s failed: %s", proxyHost, testErr)
|
||||
newOurOfflineProxies = append(newOurOfflineProxies, pxy)
|
||||
}
|
||||
return
|
||||
|
@ -84,7 +90,9 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
|||
if bv3hiErr != nil {
|
||||
okToAdd = false
|
||||
newThirdpartyBrokenProxies = append(newThirdpartyBrokenProxies, pxy)
|
||||
log.Debugf("Validate - %s failed third-party test: %s", proxyHost, bv3hiErr)
|
||||
if config.CliArgs.LogThirdPartyTest {
|
||||
log.Debugf(`%s failed third-party test for URL "%s" -- %s`, proxyHost, d, bv3hiErr)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
@ -121,8 +129,8 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
|||
}
|
||||
|
||||
p.mu.RLock()
|
||||
log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d",
|
||||
len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+len(p.thirdpartyOnlineProxies),
|
||||
log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d, Elapsed: %s",
|
||||
len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+len(p.thirdpartyOnlineProxies), time.Since(startTime),
|
||||
)
|
||||
p.mu.RUnlock()
|
||||
|
||||
|
|
Loading…
Reference in New Issue