Compare commits
17 Commits
Author | SHA1 | Date |
---|---|---|
Cyberes | e93f353860 | |
Cyberes | c3e60eaca9 | |
Cyberes | 8a2ee75188 | |
Cyberes | a886056d7f | |
Cyberes | 2ce41d0637 | |
Cyberes | ea89052ef0 | |
Cyberes | 68a444c17c | |
Cyberes | 3b66b63469 | |
Cyberes | 9316c0c1bc | |
Cyberes | d10c84d974 | |
Cyberes | 5273d1b16f | |
Cyberes | fb124561eb | |
Cyberes | dd1a880e41 | |
Cyberes | 8cddcbcb2c | |
Cyberes | 684d9c8adf | |
Cyberes | 341b82c2e4 | |
Cyberes | a38cc92700 |
25
README.md
25
README.md
|
@ -2,11 +2,11 @@
|
||||||
|
|
||||||
_A round-robin load balancer for HTTP proxies._
|
_A round-robin load balancer for HTTP proxies._
|
||||||
|
|
||||||
This is a simple proxy load balancer that will route requests to a cluster of proxy backends in a round-robin fashion.
|
This is a simple proxy load balancer that will route requests to a cluster of proxy backends in a round-robin fashion. This makes it easy to connect your clients to a large number of proxy servers without worrying about implementing anything special clientside.
|
||||||
This makes it easy to connect your clients to a large number of proxy servers without worrying about implementing
|
|
||||||
anything special clientside.
|
|
||||||
|
|
||||||
HTTPS proxy servers are not supported.
|
This proxy server will transparently forward HTTPS requests without terminating them, meaning a self-signed certificate is not required. Downstream HTTPS proxy servers are not supported.
|
||||||
|
|
||||||
|
Memory usage sits at around 25M under load.
|
||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
|
@ -15,13 +15,28 @@ HTTPS proxy servers are not supported.
|
||||||
3. Edit the config.
|
3. Edit the config.
|
||||||
4. Start the loadbalancer with `./proxy-loadbalancer --config [path to your config.yml]`
|
4. Start the loadbalancer with `./proxy-loadbalancer --config [path to your config.yml]`
|
||||||
|
|
||||||
|
## Use
|
||||||
|
|
||||||
You can run your own "public IP delivery server" `canihazip` <https://git.evulid.cc/cyberes/canihazip> or use the default `api.ipify.org`
|
You can run your own "public IP delivery server" `canihazip` <https://git.evulid.cc/cyberes/canihazip> or use the default `api.ipify.org`
|
||||||
|
|
||||||
An example systemd service `loadbalancer.service` is provided.
|
An example systemd service `loadbalancer.service` is provided.
|
||||||
|
|
||||||
|
The server displays health, stats, info at `/json`.
|
||||||
|
|
||||||
|
```
|
||||||
|
=== Proxy Load Balancer ===
|
||||||
|
Usage of ./proxy-loadbalancer:
|
||||||
|
--config [string]
|
||||||
|
Path to the config file
|
||||||
|
-d, --debug
|
||||||
|
Enable debug mode
|
||||||
|
--v Print version and exit
|
||||||
|
-h, --help Print this help message
|
||||||
|
```
|
||||||
|
|
||||||
## Special Headers
|
## Special Headers
|
||||||
|
|
||||||
The load balancer accepts special headers to control its behavior.
|
The load balancer accepts special headers to control its behavior:
|
||||||
|
|
||||||
- `Thirdparty-Bypass`: don't use any third-party endpoints for this request.
|
- `Thirdparty-Bypass`: don't use any third-party endpoints for this request.
|
||||||
- `Thirdparty-Include-Broken`: use all online endpoints for this request, including third-party ones that failed the special test.
|
- `Thirdparty-Include-Broken`: use all online endpoints for this request, including third-party ones that failed the special test.
|
||||||
|
|
|
@ -1,14 +1,17 @@
|
||||||
# Port to run on.
|
# Port to run on.
|
||||||
http_port: 9000
|
http_port: 9000
|
||||||
|
|
||||||
# How many proxies will be checked at once?
|
# How many proxies will be checked at once?
|
||||||
proxy_checkers: 50
|
proxy_checkers: 50
|
||||||
|
|
||||||
|
# The interval between proxy checks in seconds.
|
||||||
|
proxy_check_interval: 60
|
||||||
|
|
||||||
# URL to get a proxy's IP.
|
# URL to get a proxy's IP.
|
||||||
ip_checker_url: https://api.ipify.org
|
ip_checker_url: https://api.ipify.org
|
||||||
|
|
||||||
# Connection timeout for the proxies in seconds.
|
# Connection timeout for the proxies in seconds.
|
||||||
proxy_connect_timeout: 10
|
proxy_connect_timeout: 60
|
||||||
|
|
||||||
# Your proxies.
|
# Your proxies.
|
||||||
proxy_pool_ours:
|
proxy_pool_ours:
|
||||||
|
@ -27,4 +30,17 @@ thirdparty_test_urls:
|
||||||
|
|
||||||
# Don't route requests for these domains through the third-party proxies.
|
# Don't route requests for these domains through the third-party proxies.
|
||||||
thirdparty_bypass_domains:
|
thirdparty_bypass_domains:
|
||||||
- twitter.com
|
- twitter.com
|
||||||
|
|
||||||
|
# Shuffle the proxy lists whenever the background thread refreshes them.
|
||||||
|
# If false, round-robin on default order.
|
||||||
|
shuffle_proxies: false
|
||||||
|
|
||||||
|
# Don't allow requests to these domains through the proxy.
|
||||||
|
blocked_domains:
|
||||||
|
- example.com
|
||||||
|
|
||||||
|
# Resolve specific domains through specific proxies.
|
||||||
|
# Proxies here are not validated.
|
||||||
|
resolve_through:
|
||||||
|
github.com: http://1.2.3.4:3128
|
||||||
|
|
|
@ -6,7 +6,7 @@ After=network.target
|
||||||
SyslogIdentifier=proxy-loadbalancer
|
SyslogIdentifier=proxy-loadbalancer
|
||||||
User=loadbalancer
|
User=loadbalancer
|
||||||
Group=loadbalancer
|
Group=loadbalancer
|
||||||
ExecStart=/srv/loadbalancer/proxy-loadbalancer --config /etc/proxy-loadbalancer/config.yml
|
ExecStart=/srv/loadbalancer/proxy-loadbalancer -d --config /etc/proxy-loadbalancer/config.yml
|
||||||
Restart=always
|
Restart=always
|
||||||
|
|
||||||
[Install]
|
[Install]
|
||||||
|
|
|
@ -1,7 +0,0 @@
|
||||||
proxy.py @ git+https://github.com/abhinavsingh/proxy.py.git@develop
|
|
||||||
redis
|
|
||||||
requests
|
|
||||||
coloredlogs
|
|
||||||
psutil
|
|
||||||
flask
|
|
||||||
async_timeout
|
|
|
@ -18,6 +18,10 @@ type Config struct {
|
||||||
ProxyPoolThirdparty []string
|
ProxyPoolThirdparty []string
|
||||||
ThirdpartyTestUrls []string
|
ThirdpartyTestUrls []string
|
||||||
ThirdpartyBypassDomains []string
|
ThirdpartyBypassDomains []string
|
||||||
|
ShuffleProxies bool
|
||||||
|
BlockedDomains []string
|
||||||
|
ResolveThrough map[string]string
|
||||||
|
ProxyCheckInterval int
|
||||||
}
|
}
|
||||||
|
|
||||||
func SetConfig(configFile string) (*Config, error) {
|
func SetConfig(configFile string) (*Config, error) {
|
||||||
|
@ -29,11 +33,15 @@ func SetConfig(configFile string) (*Config, error) {
|
||||||
viper.SetConfigFile(configFile)
|
viper.SetConfigFile(configFile)
|
||||||
viper.SetDefault("http_port", "5000")
|
viper.SetDefault("http_port", "5000")
|
||||||
viper.SetDefault("proxy_checkers", 50)
|
viper.SetDefault("proxy_checkers", 50)
|
||||||
viper.SetDefault("proxy_connect_timeout", 10)
|
viper.SetDefault("proxy_connect_timeout", 60)
|
||||||
viper.SetDefault("proxy_pool_ours", make([]string, 0))
|
viper.SetDefault("proxy_pool_ours", make([]string, 0))
|
||||||
viper.SetDefault("proxy_pool_thirdparty", make([]string, 0))
|
viper.SetDefault("proxy_pool_thirdparty", make([]string, 0))
|
||||||
viper.SetDefault("thirdparty_test_urls", make([]string, 0))
|
viper.SetDefault("thirdparty_test_urls", make([]string, 0))
|
||||||
viper.SetDefault("thirdparty_bypass_domains", make([]string, 0))
|
viper.SetDefault("thirdparty_bypass_domains", make([]string, 0))
|
||||||
|
viper.SetDefault("shuffle_proxies", false)
|
||||||
|
viper.SetDefault("blocked_domains", make([]string, 0))
|
||||||
|
viper.SetDefault("resolve_through", make(map[string]string))
|
||||||
|
viper.SetDefault("proxy_check_interval", 60)
|
||||||
|
|
||||||
err := viper.ReadInConfig()
|
err := viper.ReadInConfig()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -48,6 +56,14 @@ func SetConfig(configFile string) (*Config, error) {
|
||||||
ProxyPoolThirdparty: viper.GetStringSlice("proxy_pool_thirdparty"),
|
ProxyPoolThirdparty: viper.GetStringSlice("proxy_pool_thirdparty"),
|
||||||
ThirdpartyTestUrls: viper.GetStringSlice("thirdparty_test_urls"),
|
ThirdpartyTestUrls: viper.GetStringSlice("thirdparty_test_urls"),
|
||||||
ThirdpartyBypassDomains: viper.GetStringSlice("thirdparty_bypass_domains"),
|
ThirdpartyBypassDomains: viper.GetStringSlice("thirdparty_bypass_domains"),
|
||||||
|
ShuffleProxies: viper.GetBool("shuffle_proxies"),
|
||||||
|
BlockedDomains: viper.GetStringSlice("blocked_domains"),
|
||||||
|
ResolveThrough: viper.GetStringMapString("resolve_through"),
|
||||||
|
ProxyCheckInterval: viper.GetInt("proxy_check_interval"),
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(config.ProxyPoolOurs) == 0 && len(config.ProxyPoolThirdparty) == 0 {
|
||||||
|
return nil, errors.New("no proxies configured")
|
||||||
}
|
}
|
||||||
|
|
||||||
if config.IpCheckerURL == "" {
|
if config.IpCheckerURL == "" {
|
||||||
|
|
|
@ -82,13 +82,18 @@ func main() {
|
||||||
log.Fatalf(`Failed to load config: %s`, err)
|
log.Fatalf(`Failed to load config: %s`, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Debugf(`Proxy check interval: %d sec`, config.GetConfig().ProxyCheckInterval)
|
||||||
|
|
||||||
proxyCluster := proxy.NewForwardProxyCluster()
|
proxyCluster := proxy.NewForwardProxyCluster()
|
||||||
go proxyCluster.ValidateProxiesThread()
|
|
||||||
proxyCluster.BalancerOnline.Wait()
|
|
||||||
go func() {
|
go func() {
|
||||||
log.Fatal(http.ListenAndServe(":"+configData.HTTPPort, proxyCluster))
|
log.Fatal(http.ListenAndServe(":"+configData.HTTPPort, proxyCluster))
|
||||||
}()
|
}()
|
||||||
log.Infof("-> Server started on 0.0.0.0:%s and accepting requests <-", configData.HTTPPort)
|
log.Infof("-> Server started on 0.0.0.0:%s <-", configData.HTTPPort)
|
||||||
|
|
||||||
|
go proxyCluster.ValidateProxiesThread()
|
||||||
|
proxyCluster.BalancerReady.Wait()
|
||||||
|
log.Infoln("-> Proxy server accepting requests <-")
|
||||||
|
|
||||||
select {}
|
select {}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -25,17 +25,35 @@ func sendRequestThroughProxy(pxy string, targetURL string) (string, error) {
|
||||||
Timeout: config.GetConfig().ProxyConnectTimeout,
|
Timeout: config.GetConfig().ProxyConnectTimeout,
|
||||||
}
|
}
|
||||||
|
|
||||||
response, err := client.Get(targetURL)
|
req, err := http.NewRequest("GET", targetURL, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
defer response.Body.Close()
|
|
||||||
if response.StatusCode == http.StatusOK {
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
|
||||||
bodyBytes, err := io.ReadAll(response.Body)
|
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||||
|
req.Header.Set("Priority", "u=0, i")
|
||||||
|
req.Header.Set("Sec-Ch-Ua", `"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"`)
|
||||||
|
req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
|
||||||
|
req.Header.Set("Sec-Ch-Ua-Platform", `"Windows"`)
|
||||||
|
req.Header.Set("Sec-Fetch-Dest", "document")
|
||||||
|
req.Header.Set("Sec-Fetch-Mode", "navigate")
|
||||||
|
req.Header.Set("Sec-Fetch-Site", "cross-site")
|
||||||
|
req.Header.Set("Sec-Fetch-User", "?1")
|
||||||
|
req.Header.Set("Upgrade-Insecure-Requests", "1")
|
||||||
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode == http.StatusOK {
|
||||||
|
bodyBytes, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
return string(bodyBytes), nil
|
return string(bodyBytes), nil
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("bad response code %d", response.StatusCode)
|
return "", fmt.Errorf("bad response code %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,6 +11,7 @@ import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"slices"
|
"slices"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -18,9 +19,19 @@ var (
|
||||||
HeaderThirdpartyBypass = "Thirdparty-Bypass"
|
HeaderThirdpartyBypass = "Thirdparty-Bypass"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time, errorMsg *string) {
|
||||||
|
msg := fmt.Sprintf(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds())
|
||||||
|
if *errorMsg == "" {
|
||||||
|
log.Infof(msg)
|
||||||
|
} else {
|
||||||
|
log.Errorf(`%s -- %s`, msg, *errorMsg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter, req *http.Request) (string, string, string, string, *url.URL, error) {
|
func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter, req *http.Request) (string, string, string, string, *url.URL, error) {
|
||||||
if p.BalancerOnline.GetCount() != 0 {
|
urlHostname := req.URL.Hostname()
|
||||||
errStr := "balancer is not ready"
|
if p.BalancerReady.GetCount() != 0 {
|
||||||
|
errStr := "proxy is not ready"
|
||||||
http.Error(w, errStr, http.StatusServiceUnavailable)
|
http.Error(w, errStr, http.StatusServiceUnavailable)
|
||||||
return "", "", "", "", nil, errors.New(errStr)
|
return "", "", "", "", nil, errors.New(errStr)
|
||||||
}
|
}
|
||||||
|
@ -34,6 +45,12 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
|
||||||
return "", "", "", "", nil, errors.New(errStr)
|
return "", "", "", "", nil, errors.New(errStr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if slices.Contains(config.GetConfig().BlockedDomains, urlHostname) {
|
||||||
|
errStr := "this domain has been blocked"
|
||||||
|
http.Error(w, errStr, http.StatusUnavailableForLegalReasons)
|
||||||
|
return "", "", "", "", nil, errors.New(errStr)
|
||||||
|
}
|
||||||
|
|
||||||
headerIncludeBrokenThirdparty := req.Header.Get(HeaderThirdpartyIncludeBroken)
|
headerIncludeBrokenThirdparty := req.Header.Get(HeaderThirdpartyIncludeBroken)
|
||||||
req.Header.Del(HeaderThirdpartyIncludeBroken)
|
req.Header.Del(HeaderThirdpartyIncludeBroken)
|
||||||
headerBypassThirdparty := req.Header.Get(HeaderThirdpartyBypass)
|
headerBypassThirdparty := req.Header.Get(HeaderThirdpartyBypass)
|
||||||
|
@ -45,15 +62,20 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
|
||||||
}
|
}
|
||||||
|
|
||||||
var selectedProxy string
|
var selectedProxy string
|
||||||
if slices.Contains(config.GetConfig().ThirdpartyBypassDomains, req.URL.Hostname()) {
|
|
||||||
selectedProxy = p.getProxyFromOurs()
|
if val, ok := config.GetConfig().ResolveThrough[urlHostname]; ok {
|
||||||
|
selectedProxy = val
|
||||||
} else {
|
} else {
|
||||||
if headerIncludeBrokenThirdparty != "" {
|
if slices.Contains(config.GetConfig().ThirdpartyBypassDomains, urlHostname) {
|
||||||
selectedProxy = p.getProxyFromAllWithBroken()
|
|
||||||
} else if headerBypassThirdparty != "" {
|
|
||||||
selectedProxy = p.getProxyFromOurs()
|
selectedProxy = p.getProxyFromOurs()
|
||||||
} else {
|
} else {
|
||||||
selectedProxy = p.getProxyFromAll()
|
if headerIncludeBrokenThirdparty != "" {
|
||||||
|
selectedProxy = p.getProxyFromAllWithBroken()
|
||||||
|
} else if headerBypassThirdparty != "" {
|
||||||
|
selectedProxy = p.getProxyFromOurs()
|
||||||
|
} else {
|
||||||
|
selectedProxy = p.getProxyFromAll()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if selectedProxy == "" {
|
if selectedProxy == "" {
|
||||||
|
@ -72,14 +94,26 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.Request) {
|
func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.Request) {
|
||||||
|
requestStartTime := time.Now()
|
||||||
|
remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
|
||||||
_, proxyUser, proxyPass, proxyHost, parsedProxyUrl, err := p.validateRequestAndGetProxy(w, req)
|
_, proxyUser, proxyPass, proxyHost, parsedProxyUrl, err := p.validateRequestAndGetProxy(w, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Error has already been handled, just log and return.
|
// Error has already been handled, just log and return.
|
||||||
log.Errorf(`Failed to validate and get proxy: "%s"`, err)
|
if proxyHost == "" {
|
||||||
|
proxyHost = "none"
|
||||||
|
}
|
||||||
|
log.Debugf(`%s -> %s -> %s -- HTTP -- Rejecting request: %s`, remoteAddr, proxyHost, req.Host, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
|
|
||||||
defer log.Debugf(`%s -> %s -> %s -- HTTP`, remoteAddr, proxyHost, req.Host)
|
// Variables for later
|
||||||
|
var returnCode *int
|
||||||
|
returnCode = new(int)
|
||||||
|
*returnCode = -1
|
||||||
|
var errorMsg *string
|
||||||
|
errorMsg = new(string)
|
||||||
|
*errorMsg = ""
|
||||||
|
defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime, errorMsg)
|
||||||
|
|
||||||
parsedProxyUrl.Scheme = "http"
|
parsedProxyUrl.Scheme = "http"
|
||||||
if proxyUser != "" && proxyPass != "" {
|
if proxyUser != "" && proxyPass != "" {
|
||||||
|
@ -94,7 +128,7 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
|
||||||
|
|
||||||
proxyReq, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
|
proxyReq, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf(`Failed to make %s request to "%s": "%s"`, req.Method, req.URL.String(), err)
|
*errorMsg = fmt.Sprintf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err)
|
||||||
http.Error(w, "failed to make request to downstream", http.StatusInternalServerError)
|
http.Error(w, "failed to make request to downstream", http.StatusInternalServerError)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -104,11 +138,12 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
|
||||||
|
|
||||||
resp, err := client.Do(proxyReq)
|
resp, err := client.Do(proxyReq)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf(`Failed to execute %s request to "%s": "%s"`, req.Method, req.URL.String(), err)
|
*errorMsg = fmt.Sprintf(`Failed to execute %s request to "%s": %s`, req.Method, req.URL.String(), err)
|
||||||
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
|
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
|
*returnCode = resp.StatusCode
|
||||||
|
|
||||||
copyHeader(w.Header(), resp.Header)
|
copyHeader(w.Header(), resp.Header)
|
||||||
w.WriteHeader(resp.StatusCode)
|
w.WriteHeader(resp.StatusCode)
|
||||||
|
@ -116,20 +151,30 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http.Request) {
|
func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http.Request) {
|
||||||
|
requestStartTime := time.Now()
|
||||||
|
remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
|
||||||
|
targetHost, _, _ := net.SplitHostPort(req.Host)
|
||||||
_, proxyUser, proxyPass, proxyHost, _, err := p.validateRequestAndGetProxy(w, req)
|
_, proxyUser, proxyPass, proxyHost, _, err := p.validateRequestAndGetProxy(w, req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Error has already been handled, just log and return.
|
// Error has already been handled, just log and return.
|
||||||
log.Errorf(`Failed to validate and get proxy: "%s"`, err)
|
if proxyHost == "" {
|
||||||
|
proxyHost = "none"
|
||||||
|
}
|
||||||
|
log.Debugf(`%s -> %s -> %s -- CONNECT -- Rejecting request: %s`, remoteAddr, proxyHost, targetHost, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
|
var returnCode *int
|
||||||
targetHost, _, _ := net.SplitHostPort(req.Host)
|
returnCode = new(int)
|
||||||
defer log.Debugf(`%s -> %s -> %s -- CONNECT`, remoteAddr, proxyHost, targetHost)
|
*returnCode = -1
|
||||||
|
var errorMsg *string
|
||||||
|
errorMsg = new(string)
|
||||||
|
*errorMsg = ""
|
||||||
|
defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime, errorMsg)
|
||||||
|
|
||||||
// Connect to the downstream proxy server instead of the target host
|
// Start a connection to the downstream proxy server.
|
||||||
proxyConn, err := net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout)
|
proxyConn, err := net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf(`Failed to dial proxy %s - %s`, proxyHost, err)
|
*errorMsg = fmt.Sprintf(`Failed to dial proxy %s - %s`, proxyHost, err)
|
||||||
http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable)
|
http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -145,38 +190,26 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
resp, err := http.ReadResponse(bufio.NewReader(proxyConn), req)
|
resp, err := http.ReadResponse(bufio.NewReader(proxyConn), req)
|
||||||
if err != nil || resp.StatusCode != 200 {
|
if resp == nil {
|
||||||
var errStr string
|
*errorMsg = fmt.Sprintf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
|
||||||
if err != nil {
|
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
|
||||||
// `err` may be nil
|
|
||||||
errStr = err.Error()
|
|
||||||
}
|
|
||||||
statusCode := -1
|
|
||||||
if resp != nil {
|
|
||||||
statusCode = resp.StatusCode
|
|
||||||
}
|
|
||||||
log.Errorf(`Failed to CONNECT to %s using proxy %s. Status code : %d - "%s"`, req.Host, proxyHost, statusCode, errStr)
|
|
||||||
|
|
||||||
// Return the original status code.
|
|
||||||
returnStatusCode := http.StatusServiceUnavailable
|
|
||||||
if statusCode != -1 {
|
|
||||||
returnStatusCode = statusCode
|
|
||||||
}
|
|
||||||
http.Error(w, "failed to execute request to downstream", returnStatusCode)
|
|
||||||
return
|
return
|
||||||
|
} else if err != nil {
|
||||||
|
log.Warnf(`Error while performing CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
|
||||||
}
|
}
|
||||||
|
*returnCode = resp.StatusCode
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
hj, ok := w.(http.Hijacker)
|
hj, ok := w.(http.Hijacker)
|
||||||
if !ok {
|
if !ok {
|
||||||
log.Errorf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
|
*errorMsg = fmt.Sprintf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
|
||||||
http.Error(w, "failed to forward connection to downstream", http.StatusServiceUnavailable)
|
http.Error(w, "failed to forward connection to downstream", http.StatusServiceUnavailable)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
clientConn, _, err := hj.Hijack()
|
clientConn, _, err := hj.Hijack()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
|
*errorMsg = fmt.Sprintf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
|
||||||
http.Error(w, "failed to execute connection forwarding to downstream", http.StatusServiceUnavailable)
|
http.Error(w, "failed to execute connection forwarding to downstream", http.StatusServiceUnavailable)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,13 +11,17 @@ import (
|
||||||
type ForwardProxyCluster struct {
|
type ForwardProxyCluster struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
ourOnlineProxies []string
|
ourOnlineProxies []string
|
||||||
|
ourOfflineProxies []string
|
||||||
thirdpartyOnlineProxies []string
|
thirdpartyOnlineProxies []string
|
||||||
thirdpartyBrokenProxies []string
|
thirdpartyBrokenProxies []string
|
||||||
|
thirdpartyOfflineProxies []string
|
||||||
ipAddresses []string
|
ipAddresses []string
|
||||||
BalancerOnline WaitGroupCountable
|
BalancerReady WaitGroupCountable
|
||||||
|
BalancerOnline bool
|
||||||
currentProxyAll int32
|
currentProxyAll int32
|
||||||
currentProxyOurs int32
|
currentProxyOurs int32
|
||||||
currentProxyAllWithBroken int32
|
currentProxyAllWithBroken int32
|
||||||
|
refreshInProgress bool
|
||||||
}
|
}
|
||||||
|
|
||||||
var log *logrus.Logger
|
var log *logrus.Logger
|
||||||
|
@ -31,23 +35,31 @@ func NewForwardProxyCluster() *ForwardProxyCluster {
|
||||||
atomic.StoreInt32(&p.currentProxyAll, 0)
|
atomic.StoreInt32(&p.currentProxyAll, 0)
|
||||||
atomic.StoreInt32(&p.currentProxyOurs, 0)
|
atomic.StoreInt32(&p.currentProxyOurs, 0)
|
||||||
atomic.StoreInt32(&p.currentProxyAllWithBroken, 0)
|
atomic.StoreInt32(&p.currentProxyAllWithBroken, 0)
|
||||||
p.BalancerOnline.Add(1)
|
p.BalancerReady.Add(1)
|
||||||
|
p.BalancerOnline = false
|
||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *ForwardProxyCluster) cycleProxy(validProxies []string, currentProxy *int32) string {
|
func (p *ForwardProxyCluster) cycleProxy(validProxies []string, currentProxy *int32) string {
|
||||||
// Just round robin
|
// Just round robin
|
||||||
currProxy := atomic.LoadInt32(currentProxy)
|
p.mu.RLock()
|
||||||
|
defer p.mu.RUnlock()
|
||||||
|
currProxy := int(atomic.LoadInt32(currentProxy))
|
||||||
|
if currProxy > len(validProxies)-1 {
|
||||||
|
// This might happen if the background thread is currently making changes to the list of proxies.
|
||||||
|
// Just set it to the current length of the proxies array and move on.
|
||||||
|
currProxy = len(validProxies) - 1
|
||||||
|
}
|
||||||
downstreamProxy := validProxies[currProxy]
|
downstreamProxy := validProxies[currProxy]
|
||||||
newCurrentProxy := (currProxy + 1) % int32(len(validProxies))
|
newCurrentProxy := (currProxy + 1) % len(validProxies)
|
||||||
atomic.StoreInt32(currentProxy, newCurrentProxy)
|
atomic.StoreInt32(currentProxy, int32(newCurrentProxy))
|
||||||
return downstreamProxy
|
return downstreamProxy
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *ForwardProxyCluster) getProxyFromAll() string {
|
func (p *ForwardProxyCluster) getProxyFromAll() string {
|
||||||
p.mu.RLock()
|
p.mu.RLock()
|
||||||
defer p.mu.RUnlock()
|
defer p.mu.RUnlock()
|
||||||
validProxies := removeDuplicates(append(p.ourOnlineProxies, p.thirdpartyOnlineProxies...))
|
validProxies := append(p.ourOnlineProxies, p.thirdpartyOnlineProxies...)
|
||||||
return p.cycleProxy(validProxies, &p.currentProxyAll)
|
return p.cycleProxy(validProxies, &p.currentProxyAll)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -61,7 +73,7 @@ func (p *ForwardProxyCluster) getProxyFromOurs() string {
|
||||||
func (p *ForwardProxyCluster) getProxyFromAllWithBroken() string {
|
func (p *ForwardProxyCluster) getProxyFromAllWithBroken() string {
|
||||||
p.mu.RLock()
|
p.mu.RLock()
|
||||||
defer p.mu.RUnlock()
|
defer p.mu.RUnlock()
|
||||||
validProxies := removeDuplicates(slices.Concat(p.ourOnlineProxies, p.thirdpartyBrokenProxies, p.thirdpartyOnlineProxies))
|
validProxies := slices.Concat(p.ourOnlineProxies, p.thirdpartyBrokenProxies, p.thirdpartyOnlineProxies)
|
||||||
return p.cycleProxy(validProxies, &p.currentProxyAllWithBroken)
|
return p.cycleProxy(validProxies, &p.currentProxyAllWithBroken)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,26 +1,90 @@
|
||||||
package proxy
|
package proxy
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var startTime time.Time
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
startTime = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
func (p *ForwardProxyCluster) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
func (p *ForwardProxyCluster) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||||
if req.Method == http.MethodConnect {
|
if req.Method == http.MethodConnect {
|
||||||
// HTTPS
|
// HTTPS
|
||||||
p.proxyHttpsConnect(w, req)
|
p.proxyHttpsConnect(w, req)
|
||||||
} else {
|
} else {
|
||||||
// HTTP
|
// HTTP
|
||||||
if req.URL.Scheme != "http" {
|
if req.URL.Scheme == "" {
|
||||||
//msg := fmt.Sprintf(`unsupported protocal "%s"`, req.URL.Scheme)
|
// When the client connects using the server as a web server.
|
||||||
//log.Errorf(msg)
|
remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
|
||||||
//http.Error(w, msg, http.StatusBadRequest)
|
defer log.Infof(`%s -- %s`, remoteAddr, req.URL.Path)
|
||||||
rand.New(rand.NewSource(time.Now().Unix()))
|
if req.URL.Path == "/" {
|
||||||
fmt.Fprint(w, "proxy-loadbalancer\n<https://git.evulid.cc/cyberes/proxy-loadbalancer>\n\n"+retardation[rand.Intn(len(retardation))])
|
rand.New(rand.NewSource(time.Now().Unix()))
|
||||||
return
|
fmt.Fprint(w, "proxy-loadbalancer <https://git.evulid.cc/cyberes/proxy-loadbalancer>\nSee /json for status info.\n\n\n\n"+retardation[rand.Intn(len(retardation))])
|
||||||
|
return
|
||||||
|
} else if req.URL.Path == "/json" {
|
||||||
|
p.mu.RLock()
|
||||||
|
response := map[string]interface{}{
|
||||||
|
"uptime": int(math.Round(time.Since(startTime).Seconds())),
|
||||||
|
"online": p.BalancerOnline && p.BalancerReady.GetCount() == 0,
|
||||||
|
"refreshInProgress": p.refreshInProgress,
|
||||||
|
"proxies": map[string]interface{}{
|
||||||
|
"totalOnline": len(p.ourOnlineProxies) + len(p.thirdpartyOnlineProxies),
|
||||||
|
"ours": map[string]interface{}{
|
||||||
|
"online": removeCredentials(p.ourOnlineProxies),
|
||||||
|
"offline": removeCredentials(p.ourOfflineProxies),
|
||||||
|
},
|
||||||
|
"thirdParty": map[string]interface{}{
|
||||||
|
"online": removeCredentials(p.thirdpartyOnlineProxies),
|
||||||
|
"broken": removeCredentials(p.thirdpartyBrokenProxies),
|
||||||
|
"offline": removeCredentials(p.thirdpartyOfflineProxies),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
p.mu.RUnlock()
|
||||||
|
jsonResponse, err := json.MarshalIndent(response, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
log.Errorln(err)
|
||||||
|
http.Error(w, "Path not found", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Cache-Control", "no-store")
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.Write(jsonResponse)
|
||||||
|
return
|
||||||
|
} else {
|
||||||
|
http.Error(w, "Path not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// When the client connects using the server as a proxy.
|
||||||
|
p.proxyHttpConnect(w, req)
|
||||||
}
|
}
|
||||||
p.proxyHttpConnect(w, req)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func removeCredentials(proxyURLs []string) []string {
|
||||||
|
var newURLs []string
|
||||||
|
for _, proxyURL := range proxyURLs {
|
||||||
|
u, err := url.Parse(proxyURL)
|
||||||
|
if err != nil {
|
||||||
|
// Skip if invalid.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
u.User = nil
|
||||||
|
newURLs = append(newURLs, u.String())
|
||||||
|
}
|
||||||
|
if len(newURLs) == 0 {
|
||||||
|
newURLs = make([]string, 0)
|
||||||
|
}
|
||||||
|
return newURLs
|
||||||
|
}
|
||||||
|
|
|
@ -17,22 +17,29 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
||||||
ctx := context.TODO()
|
ctx := context.TODO()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
// TODO: need to have these be temp vars and then copy them over when finished
|
startTime := time.Now()
|
||||||
|
p.refreshInProgress = true
|
||||||
allProxies := removeDuplicates(append(config.GetConfig().ProxyPoolOurs, config.GetConfig().ProxyPoolThirdparty...))
|
allProxies := removeDuplicates(append(config.GetConfig().ProxyPoolOurs, config.GetConfig().ProxyPoolThirdparty...))
|
||||||
p.ourOnlineProxies = make([]string, 0)
|
newOurOnlineProxies := make([]string, 0)
|
||||||
p.thirdpartyOnlineProxies = make([]string, 0)
|
newOurOfflineProxies := make([]string, 0)
|
||||||
p.thirdpartyBrokenProxies = make([]string, 0)
|
newThirdpartyOnlineProxies := make([]string, 0)
|
||||||
p.ipAddresses = make([]string, 0)
|
newThirdpartyBrokenProxies := make([]string, 0)
|
||||||
|
newThirdpartyOfflineProxies := make([]string, 0)
|
||||||
|
newIpAddresses := make([]string, 0)
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
for _, pxy := range allProxies {
|
for _, pxy := range allProxies {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
// TODO: semaphore to limit active checks
|
|
||||||
go func(pxy string) {
|
go func(pxy string) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
if err := sem.Acquire(ctx, 1); err != nil {
|
if err := sem.Acquire(ctx, 1); err != nil {
|
||||||
log.Errorf("Validate - failed to acquire semaphore: %v\n", err)
|
log.Errorf("Validate - failed to acquire semaphore: %v", err)
|
||||||
|
if isThirdparty(pxy) {
|
||||||
|
newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
|
||||||
|
} else {
|
||||||
|
newOurOfflineProxies = append(newOurOfflineProxies, pxy)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer sem.Release(1)
|
defer sem.Release(1)
|
||||||
|
@ -40,58 +47,88 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
||||||
_, _, proxyHost, _, err := splitProxyURL(pxy)
|
_, _, proxyHost, _, err := splitProxyURL(pxy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf(`Invalid proxy "%s"`, pxy)
|
log.Errorf(`Invalid proxy "%s"`, pxy)
|
||||||
|
if isThirdparty(pxy) {
|
||||||
|
newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
|
||||||
|
} else {
|
||||||
|
newOurOfflineProxies = append(newOurOfflineProxies, pxy)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Test the proxy.
|
// Test the proxy.
|
||||||
ipAddr, testErr := sendRequestThroughProxy(pxy, config.GetConfig().IpCheckerURL)
|
ipAddr, testErr := sendRequestThroughProxy(pxy, config.GetConfig().IpCheckerURL)
|
||||||
if testErr != nil {
|
if testErr != nil {
|
||||||
log.Warnf("Validate - proxy %s failed: %s", proxyHost, testErr)
|
log.Debugf("Validate - %s failed: %s", proxyHost, testErr)
|
||||||
|
if isThirdparty(pxy) {
|
||||||
|
newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
|
||||||
|
} else {
|
||||||
|
newOurOfflineProxies = append(newOurOfflineProxies, pxy)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if slices.Contains(p.ipAddresses, ipAddr) {
|
if slices.Contains(newIpAddresses, ipAddr) {
|
||||||
log.Warnf("Validate - duplicate IP Address %s for proxy %s", ipAddr, proxyHost)
|
log.Warnf("Validate - duplicate IP Address %s for %s", ipAddr, proxyHost)
|
||||||
|
if isThirdparty(pxy) {
|
||||||
|
newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
|
||||||
|
} else {
|
||||||
|
newOurOfflineProxies = append(newOurOfflineProxies, pxy)
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
p.ipAddresses = append(p.ipAddresses, ipAddr)
|
newIpAddresses = append(newIpAddresses, ipAddr)
|
||||||
|
|
||||||
// Sort the proxy into the right groups.
|
// Sort the proxy into the right groups.
|
||||||
if isThirdparty(pxy) {
|
if isThirdparty(pxy) {
|
||||||
p.mu.Lock()
|
okToAdd := true
|
||||||
p.thirdpartyOnlineProxies = append(p.thirdpartyOnlineProxies, pxy)
|
|
||||||
p.mu.Unlock()
|
|
||||||
|
|
||||||
for _, d := range config.GetConfig().ThirdpartyTestUrls {
|
for _, d := range config.GetConfig().ThirdpartyTestUrls {
|
||||||
_, bv3hiErr := sendRequestThroughProxy(pxy, d)
|
_, bv3hiErr := sendRequestThroughProxy(pxy, d)
|
||||||
if bv3hiErr != nil {
|
if bv3hiErr != nil {
|
||||||
log.Debugf("Validate - Third-party %s failed: %s\n", proxyHost, bv3hiErr)
|
okToAdd = false
|
||||||
p.thirdpartyBrokenProxies = append(p.thirdpartyBrokenProxies, pxy)
|
newThirdpartyBrokenProxies = append(newThirdpartyBrokenProxies, pxy)
|
||||||
|
log.Debugf("Validate - %s failed third-party test: %s", proxyHost, bv3hiErr)
|
||||||
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if okToAdd {
|
||||||
|
newThirdpartyOnlineProxies = append(newThirdpartyOnlineProxies, pxy)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
p.mu.Lock()
|
newOurOnlineProxies = append(newOurOnlineProxies, pxy)
|
||||||
p.ourOnlineProxies = append(p.ourOnlineProxies, pxy)
|
|
||||||
p.mu.Unlock()
|
|
||||||
}
|
}
|
||||||
}(pxy)
|
}(pxy)
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
if !started {
|
p.mu.Lock()
|
||||||
|
p.ourOnlineProxies = removeDuplicates(newOurOnlineProxies)
|
||||||
|
p.ourOfflineProxies = newOurOfflineProxies
|
||||||
|
p.thirdpartyOnlineProxies = removeDuplicates(newThirdpartyOnlineProxies)
|
||||||
|
p.thirdpartyBrokenProxies = removeDuplicates(newThirdpartyBrokenProxies)
|
||||||
|
p.thirdpartyOfflineProxies = newThirdpartyOfflineProxies
|
||||||
|
p.ipAddresses = removeDuplicates(newIpAddresses)
|
||||||
|
p.BalancerOnline = len(slices.Concat(p.ourOnlineProxies, p.thirdpartyOnlineProxies, p.thirdpartyBrokenProxies)) > 0 // Online only if there are active and online proxies.
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
if config.GetConfig().ShuffleProxies {
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
p.ourOnlineProxies = shuffle(p.ourOnlineProxies)
|
p.ourOnlineProxies = shuffle(p.ourOnlineProxies)
|
||||||
p.thirdpartyOnlineProxies = shuffle(p.thirdpartyOnlineProxies)
|
p.thirdpartyOnlineProxies = shuffle(p.thirdpartyOnlineProxies)
|
||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
if !started {
|
||||||
started = true
|
started = true
|
||||||
p.BalancerOnline.Done()
|
p.BalancerReady.Done()
|
||||||
}
|
}
|
||||||
|
|
||||||
p.mu.RLock()
|
p.mu.RLock()
|
||||||
log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d\n",
|
log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d, Elapsed: %s",
|
||||||
len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+(len(p.thirdpartyOnlineProxies)-len(p.thirdpartyBrokenProxies)))
|
len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+len(p.thirdpartyOnlineProxies), time.Since(startTime),
|
||||||
|
)
|
||||||
p.mu.RUnlock()
|
p.mu.RUnlock()
|
||||||
|
|
||||||
time.Sleep(60 * time.Second)
|
p.refreshInProgress = false
|
||||||
|
time.Sleep(time.Duration(config.GetConfig().ProxyCheckInterval) * time.Second)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -115,12 +152,9 @@ func removeDuplicates(elements []string) []string {
|
||||||
if encountered[elements[v]] == true {
|
if encountered[elements[v]] == true {
|
||||||
// Do not add duplicate.
|
// Do not add duplicate.
|
||||||
} else {
|
} else {
|
||||||
// Record this element as an encountered element.
|
|
||||||
encountered[elements[v]] = true
|
encountered[elements[v]] = true
|
||||||
// Append to result slice.
|
|
||||||
result = append(result, elements[v])
|
result = append(result, elements[v])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Return the new slice.
|
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue