finish prototype

This commit is contained in:
Cyberes 2024-04-12 18:23:18 -06:00
parent d1bea64203
commit 946f2d7a11
16 changed files with 537 additions and 102 deletions

2
.gitignore vendored
View File

@ -3,6 +3,8 @@ config.py
dist/
test.go
config.yml
config.yaml
# ---> Python
# Byte-compiled / optimized / DLL files

View File

@ -9,10 +9,10 @@ else
fi
mkdir -p "$SCRIPT_DIR/dist"
rm "$SCRIPT_DIR"/dist/crazyfs-* &> /dev/null
rm "$SCRIPT_DIR"/dist/* &> /dev/null
BUILDARGS="$(uname)-$(uname -p)"
OUTPUTFILE="$SCRIPT_DIR/dist/crazyfs-$VERSION-$BUILDARGS"
OUTPUTFILE="$SCRIPT_DIR/dist/proxy-loadbalancer-$VERSION-$BUILDARGS"
cd "$SCRIPT_DIR/src" || exit 1
go mod tidy

30
config.example.yml Normal file
View File

@ -0,0 +1,30 @@
# Port to run on.
http_port: 9000
# How many proxies will be checked at once?
proxy_checkers: 50
# URL to get a proxy's IP.
ip_checker_url: https://api.ipify.org
# Connection timeout for the proxies in seconds.
proxy_connect_timeout: 10
# Your proxies.
proxy_pool_ours:
- http://1.2.3.4:3128
- http://5.6.7.8:3128
# Your third-party proxies.
proxy_pool_thirdparty:
- http://username:password@example:10001
# URL used to test third-party proxies against.
# Some proxies just don't work on some domains. If a proxy fails this check it will be marked as
# "unhealthy" and removed from the general pool.
thirdparty_test_urls:
- https://files.catbox.moe/1hvrlj.png
# Don't route requests for these domains through the third-party proxies.
thirdparty_bypass_domains:
- twitter.com

82
src/config/config.go Normal file
View File

@ -0,0 +1,82 @@
package config
import (
"errors"
"github.com/spf13/viper"
"time"
)
// The global, read-only config variable.
var cfg *Config
type Config struct {
HTTPPort string
IpCheckerURL string
MaxProxyCheckers int
ProxyConnectTimeout time.Duration
ProxyPoolOurs []string
ProxyPoolThirdparty []string
ThirdpartyTestUrls []string
ThirdpartyBypassDomains []string
}
func SetConfig(configFile string) (*Config, error) {
// Only allow the config to be set once.
if cfg != nil {
panic("Config has already been set!")
}
viper.SetConfigFile(configFile)
viper.SetDefault("http_port", "5000")
viper.SetDefault("proxy_checkers", 50)
viper.SetDefault("proxy_connect_timeout", 10)
viper.SetDefault("proxy_pool_ours", make([]string, 0))
viper.SetDefault("proxy_pool_thirdparty", make([]string, 0))
viper.SetDefault("thirdparty_test_urls", make([]string, 0))
viper.SetDefault("thirdparty_bypass_domains", make([]string, 0))
err := viper.ReadInConfig()
if err != nil {
return nil, err
}
config := &Config{
HTTPPort: viper.GetString("http_port"),
IpCheckerURL: viper.GetString("ip_checker_url"),
MaxProxyCheckers: viper.GetInt("proxy_checkers"),
ProxyPoolOurs: viper.GetStringSlice("proxy_pool_ours"),
ProxyPoolThirdparty: viper.GetStringSlice("proxy_pool_thirdparty"),
ThirdpartyTestUrls: viper.GetStringSlice("thirdparty_test_urls"),
ThirdpartyBypassDomains: viper.GetStringSlice("thirdparty_bypass_domains"),
}
if config.IpCheckerURL == "" {
return nil, errors.New("ip_checker_url is required")
}
timeout := viper.GetInt("proxy_connect_timeout")
if timeout <= 0 {
return nil, errors.New("proxy_connect_timeout must be greater than 0")
}
config.ProxyConnectTimeout = time.Duration(timeout) * time.Second
proxyPoolOursErr := validateProxies(config.ProxyPoolOurs)
if proxyPoolOursErr != nil {
return nil, proxyPoolOursErr
}
proxyPoolThirdpartyErr := validateProxies(config.ProxyPoolThirdparty)
if proxyPoolThirdpartyErr != nil {
return nil, proxyPoolThirdpartyErr
}
cfg = config
return config, nil
}
func GetConfig() *Config {
if cfg == nil {
panic("Config has not been set!")
}
return cfg
}

View File

@ -6,10 +6,10 @@ import (
"strings"
)
func ValidateProxies(proxies []string) error {
func validateProxies(proxies []string) error {
for _, proxy := range proxies {
if !strings.HasPrefix("http://", proxy) {
return errors.New(fmt.Sprintf(`proxy "%s" must start with http://`, proxy))
if !strings.HasPrefix(proxy, "http://") {
return errors.New(fmt.Sprintf(`Proxy URLs must start with "http://" - "%s"`, proxy))
}
}
return nil

View File

@ -3,8 +3,29 @@ module main
go 1.22.1
require (
github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5
github.com/sirupsen/logrus v1.9.3
github.com/spf13/viper v1.18.2
golang.org/x/sync v0.7.0
)
require github.com/elazarl/goproxy/ext v0.0.0-20231117061959-7cc037d33fb5 // indirect
require (
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View File

@ -1,8 +1,75 @@
github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5 h1:m62nsMU279qRD9PQSWD1l66kmkXzuYcnVJqL4XLeV2M=
github.com/elazarl/goproxy v0.0.0-20231117061959-7cc037d33fb5/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
github.com/elazarl/goproxy/ext v0.0.0-20231117061959-7cc037d33fb5 h1:iGoePcl8bIDJxxRAL2Q4E4Rt35z5m917RJb8lAvdrQw=
github.com/elazarl/goproxy/ext v0.0.0-20231117061959-7cc037d33fb5/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ=
github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

27
src/logging/logging.go Normal file
View File

@ -0,0 +1,27 @@
package logging
import (
"github.com/sirupsen/logrus"
)
var log *logrus.Logger
func init() {
log = logrus.New()
// Set log output format
customFormatter := new(logrus.TextFormatter)
customFormatter.TimestampFormat = "2006-01-02 15:04:05"
customFormatter.FullTimestamp = true
log.SetFormatter(customFormatter)
}
// InitLogger initializes the global logger with the specified log level
func InitLogger(logLevel logrus.Level) {
log.SetLevel(logLevel)
}
// GetLogger returns the global logger instance
func GetLogger() *logrus.Logger {
return log
}

View File

@ -3,10 +3,13 @@ package main
import (
"flag"
"fmt"
"log"
"github.com/sirupsen/logrus"
"main/config"
"main/logging"
"main/proxy"
"net/http"
"os"
"path/filepath"
"runtime/debug"
)
@ -48,13 +51,44 @@ func main() {
os.Exit(0)
}
if cliArgs.debug {
logging.InitLogger(logrus.DebugLevel)
} else {
logging.InitLogger(logrus.InfoLevel)
}
log := logging.GetLogger()
log.Debugln("Initializing...")
if cliArgs.configFile == "" {
exePath, err := os.Executable()
if err != nil {
panic(err)
}
exeDir := filepath.Dir(exePath)
if _, err := os.Stat(filepath.Join(exeDir, "config.yml")); err == nil {
if _, err := os.Stat(filepath.Join(exeDir, "config.yaml")); err == nil {
log.Fatalln("Both config.yml and config.yaml exist in the executable directory. Please specify one with the --config flag.")
}
cliArgs.configFile = filepath.Join(exeDir, "config.yml")
} else if _, err := os.Stat(filepath.Join(exeDir, "config.yaml")); err == nil {
cliArgs.configFile = filepath.Join(exeDir, "config.yaml")
} else {
log.Fatalln("No config file found in the executable directory. Please provide one with the --config flag.")
}
}
configData, err := config.SetConfig(cliArgs.configFile)
if err != nil {
log.Fatalf(`Failed to load config: %s`, err)
}
proxyCluster := proxy.NewForwardProxyCluster()
go proxyCluster.ValidateProxiesThread()
proxyCluster.BalancerOnline.Wait()
go func() {
log.Fatal(http.ListenAndServe(":5000", proxyCluster))
log.Fatal(http.ListenAndServe(":"+configData.HTTPPort, proxyCluster))
}()
fmt.Println("Server started!")
log.Infof("-> Server started on 0.0.0.0:%s and accepting requests <-", configData.HTTPPort)
select {}
}

View File

@ -3,27 +3,26 @@ package proxy
import (
"fmt"
"io"
"main/config"
"net/http"
"net/url"
"time"
)
func sendRequestThroughProxy(proxyUrl string, targetURL string) (string, error) {
parsedProxyUrl, err := url.Parse(proxyUrl)
func sendRequestThroughProxy(pxy string, targetURL string) (string, error) {
proxyUser, proxyPass, _, parsedProxyUrl, err := splitProxyURL(pxy)
if err != nil {
return "", err
}
if IsSmartproxy(proxyUrl) {
// Set the username and password for proxy authentication if Smartproxy
parsedProxyUrl.User = url.UserPassword(smartproxyUsername, smartproxyPassword)
if proxyUser != "" && proxyPass != "" {
parsedProxyUrl.User = url.UserPassword(proxyUser, proxyPass)
}
transport := &http.Transport{
Proxy: http.ProxyURL(parsedProxyUrl),
}
client := &http.Client{
Transport: transport,
Timeout: time.Second * 10,
Timeout: config.GetConfig().ProxyConnectTimeout,
}
response, err := client.Get(targetURL)

View File

@ -2,28 +2,95 @@ package proxy
import (
"bufio"
"encoding/base64"
"errors"
"fmt"
"io"
"log"
"main/config"
"net"
"net/http"
"net/url"
"strings"
"sync/atomic"
"slices"
)
func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.Request) {
proxyURLParsed, _ := url.Parse(p.getProxy())
proxyURLParsed.Scheme = "http"
func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter, req *http.Request) (string, string, string, string, *url.URL, error) {
if p.BalancerOnline.GetCount() != 0 {
errStr := "balancer is not ready"
http.Error(w, errStr, http.StatusServiceUnavailable)
return "", "", "", "", nil, errors.New(errStr)
}
p.mu.RLock()
defer p.mu.RUnlock()
if len(p.ourOnlineProxies) == 0 && len(p.thirdpartyOnlineProxies) == 0 {
errStr := "no valid backends"
http.Error(w, errStr, http.StatusServiceUnavailable)
return "", "", "", "", nil, errors.New(errStr)
}
headerIncludeBrokenThirdparty := req.Header.Get("Thirdparty-Include-Broken")
req.Header.Del("Thirdparty-Include-Broken")
headerBypassThirdparty := req.Header.Get("Thirdparty-Bypass")
req.Header.Del("Thirdparty-Bypass")
if headerBypassThirdparty != "" && headerIncludeBrokenThirdparty != "" {
errStr := "duplicate options headers detected, rejecting request"
http.Error(w, errStr, http.StatusBadRequest)
return "", "", "", "", nil, errors.New(errStr)
}
var selectedProxy string
if slices.Contains(config.GetConfig().ThirdpartyBypassDomains, req.URL.Hostname()) {
selectedProxy = p.getProxyFromOurs()
} else {
if headerIncludeBrokenThirdparty != "" {
selectedProxy = p.getProxyFromAllWithBroken()
} else if headerBypassThirdparty != "" {
selectedProxy = p.getProxyFromOurs()
} else {
selectedProxy = p.getProxyFromAll()
}
}
if selectedProxy == "" {
panic("selected proxy was empty!")
}
proxyUser, proxyPass, proxyHost, parsedProxyUrl, err := splitProxyURL(selectedProxy)
if err != nil {
errStr := "failed to parse downstream proxy assignment"
http.Error(w, errStr, http.StatusBadRequest)
return "", "", "", "", nil, errors.New(fmt.Sprintf(`%s: %s`, errStr, err.Error()))
}
return selectedProxy, proxyUser, proxyPass, proxyHost, parsedProxyUrl, nil
}
func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.Request) {
_, proxyUser, proxyPass, proxyHost, parsedProxyUrl, err := p.validateRequestAndGetProxy(w, req)
if err != nil {
// Error has already been handled, just log and return.
log.Errorf(`Failed to validate and get proxy: "%s"`, err)
return
}
remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
defer log.Debugf(`%s -> %s -> %s -- HTTP`, remoteAddr, proxyHost, req.Host)
parsedProxyUrl.Scheme = "http"
if proxyUser != "" && proxyPass != "" {
parsedProxyUrl.User = url.UserPassword(proxyUser, proxyPass)
}
client := &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyURL(proxyURLParsed),
Proxy: http.ProxyURL(parsedProxyUrl),
},
Timeout: config.GetConfig().ProxyConnectTimeout,
}
proxyReq, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(`Failed to make %s request to "%s": "%s"`, req.Method, req.URL.String(), err)
http.Error(w, "failed to make request to downstream", http.StatusInternalServerError)
return
}
@ -32,7 +99,8 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
resp, err := client.Do(proxyReq)
if err != nil {
http.Error(w, err.Error(), http.StatusServiceUnavailable)
log.Errorf(`Failed to execute %s request to "%s": "%s"`, req.Method, req.URL.String(), err)
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
return
}
defer resp.Body.Close()
@ -42,49 +110,72 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
io.Copy(w, resp.Body)
}
func (p *ForwardProxyCluster) proxyHTTPSConnect(w http.ResponseWriter, req *http.Request) {
log.Printf("CONNECT requested to %v (from %v)", req.Host, req.RemoteAddr)
allValidProxies := append(p.ourOnlineProxies, p.smartproxyOnlineProxies...)
currentProxy := atomic.LoadInt32(&p.CurrentProxy)
downstreamProxy := allValidProxies[currentProxy]
downstreamProxy = strings.Replace(downstreamProxy, "http://", "", -1)
downstreamProxy = strings.Replace(downstreamProxy, "https://", "", -1)
newCurrentProxy := (currentProxy + 1) % int32(len(testProxies))
atomic.StoreInt32(&p.CurrentProxy, newCurrentProxy)
func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http.Request) {
_, proxyUser, proxyPass, proxyHost, _, err := p.validateRequestAndGetProxy(w, req)
if err != nil {
// Error has already been handled, just log and return.
log.Errorf(`Failed to validate and get proxy: "%s"`, err)
return
}
remoteAddr, _, _ := net.SplitHostPort(req.RemoteAddr)
targetHost, _, _ := net.SplitHostPort(req.Host)
defer log.Debugf(`%s -> %s -> %s -- CONNECT`, remoteAddr, proxyHost, targetHost)
// Connect to the downstream proxy server instead of the target host
proxyConn, err := net.Dial("tcp", downstreamProxy)
proxyConn, err := net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout)
if err != nil {
log.Println("failed to dial to proxy", downstreamProxy, err)
http.Error(w, err.Error(), http.StatusServiceUnavailable)
log.Errorf(`Failed to dial proxy %s - %s`, proxyHost, err)
http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable)
return
}
// Proxy authentication
auth := fmt.Sprintf("%s:%s", proxyUser, proxyPass)
encodedAuth := base64.StdEncoding.EncodeToString([]byte(auth))
authHeader := "Proxy-Authorization: Basic " + encodedAuth
// Send a new CONNECT request to the downstream proxy
_, err = fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", req.Host, req.Host)
_, err = fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n%s\r\n\r\n", req.Host, req.Host, authHeader)
if err != nil {
return
}
resp, err := http.ReadResponse(bufio.NewReader(proxyConn), req)
if err != nil || resp.StatusCode != 200 {
log.Println("failed to CONNECT to target", req.Host)
http.Error(w, err.Error(), http.StatusServiceUnavailable)
var errStr string
if err != nil {
// `err` may be nil
errStr = err.Error()
}
statusCode := -1
if resp != nil {
statusCode = resp.StatusCode
}
log.Errorf(`Failed to CONNECT to %s using proxy %s. Status code : %d - "%s"`, req.Host, proxyHost, statusCode, errStr)
// Return the original status code.
returnStatusCode := http.StatusServiceUnavailable
if statusCode != -1 {
returnStatusCode = statusCode
}
http.Error(w, "failed to execute request to downstream", returnStatusCode)
return
}
w.WriteHeader(http.StatusOK)
hj, ok := w.(http.Hijacker)
if !ok {
log.Fatal("http server doesn't support hijacking connection")
log.Errorf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
http.Error(w, "failed to forward connection to downstream", http.StatusServiceUnavailable)
return
}
clientConn, _, err := hj.Hijack()
if err != nil {
log.Fatal("http hijacking failed")
log.Errorf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
http.Error(w, "failed to execute connection forwarding to downstream", http.StatusServiceUnavailable)
return
}
log.Println("tunnel established")
go tunnelConn(proxyConn, clientConn)
go tunnelConn(clientConn, proxyConn)
}

View File

@ -1,35 +1,67 @@
package proxy
import (
"github.com/sirupsen/logrus"
"main/logging"
"slices"
"sync"
"sync/atomic"
)
type ForwardProxyCluster struct {
// TODO: mutex rwlock
mu sync.RWMutex
ourOnlineProxies []string
smartproxyOnlineProxies []string
smartproxyBrokenProxies []string
thirdpartyOnlineProxies []string
thirdpartyBrokenProxies []string
ipAddresses []string
BalancerOnline sync.WaitGroup
CurrentProxy int32
BalancerOnline WaitGroupCountable
currentProxyAll int32
currentProxyOurs int32
currentProxyAllWithBroken int32
}
// TODO: move all smartproxy things to "thirdparty"
var log *logrus.Logger
func init() {
log = logging.GetLogger()
}
func NewForwardProxyCluster() *ForwardProxyCluster {
p := &ForwardProxyCluster{}
atomic.StoreInt32(&p.CurrentProxy, 0)
atomic.StoreInt32(&p.currentProxyAll, 0)
atomic.StoreInt32(&p.currentProxyOurs, 0)
atomic.StoreInt32(&p.currentProxyAllWithBroken, 0)
p.BalancerOnline.Add(1)
return p
}
func (p *ForwardProxyCluster) getProxy() string {
func (p *ForwardProxyCluster) cycleProxy(validProxies []string, currentProxy *int32) string {
// Just round robin
allValidProxies := append(p.ourOnlineProxies, p.smartproxyOnlineProxies...)
currentProxy := atomic.LoadInt32(&p.CurrentProxy)
downstreamProxy := allValidProxies[currentProxy]
newCurrentProxy := (currentProxy + 1) % int32(len(testProxies))
atomic.StoreInt32(&p.CurrentProxy, newCurrentProxy)
currProxy := atomic.LoadInt32(currentProxy)
downstreamProxy := validProxies[currProxy]
newCurrentProxy := (currProxy + 1) % int32(len(validProxies))
atomic.StoreInt32(currentProxy, newCurrentProxy)
return downstreamProxy
}
func (p *ForwardProxyCluster) getProxyFromAll() string {
p.mu.RLock()
defer p.mu.RUnlock()
validProxies := removeDuplicates(append(p.ourOnlineProxies, p.thirdpartyOnlineProxies...))
return p.cycleProxy(validProxies, &p.currentProxyAll)
}
func (p *ForwardProxyCluster) getProxyFromOurs() string {
p.mu.RLock()
defer p.mu.RUnlock()
validProxies := p.ourOnlineProxies
return p.cycleProxy(validProxies, &p.currentProxyOurs)
}
func (p *ForwardProxyCluster) getProxyFromAllWithBroken() string {
p.mu.RLock()
defer p.mu.RUnlock()
validProxies := removeDuplicates(slices.Concat(p.ourOnlineProxies, p.thirdpartyBrokenProxies, p.thirdpartyOnlineProxies))
return p.cycleProxy(validProxies, &p.currentProxyAllWithBroken)
}

View File

@ -2,20 +2,19 @@ package proxy
import (
"fmt"
"log"
"net/http"
)
func (p *ForwardProxyCluster) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if req.Method == http.MethodConnect {
// HTTPS
p.proxyHTTPSConnect(w, req)
p.proxyHttpsConnect(w, req)
} else {
// HTTP
if req.URL.Scheme != "http" {
msg := fmt.Sprintf(`unsupported protocal "%s"`, req.URL.Scheme)
log.Errorf(msg)
http.Error(w, msg, http.StatusBadRequest)
log.Println(msg)
return
}
p.proxyHttpConnect(w, req)

View File

@ -1,14 +1,36 @@
package proxy
import (
"main/config"
"net/url"
"slices"
"strings"
)
func IsSmartproxy(proxyUrl string) bool {
parsedProxyUrl, err := url.Parse(proxyUrl)
func splitProxyURL(proxyURL string) (string, string, string, *url.URL, error) {
u, err := url.Parse(proxyURL)
if err != nil {
panic(err)
return "", "", "", nil, err
}
return strings.Split(parsedProxyUrl.Host, ":")[0] == "dc.smartproxy.com"
var username, password string
if u.User != nil {
username = u.User.Username()
password, _ = u.User.Password()
}
host := u.Host
return username, password, host, u, nil
}
func isThirdparty(proxyUrl string) bool {
return slices.Contains(config.GetConfig().ProxyPoolThirdparty, proxyUrl)
}
func stripHTTP(url string) string {
var newStr string
newStr = strings.Replace(url, "http://", "", -1)
newStr = strings.Replace(newStr, "https://", "", -1)
return newStr
}

View File

@ -2,9 +2,8 @@ package proxy
import (
"context"
"fmt"
"golang.org/x/sync/semaphore"
"log"
"main/config"
"math/rand"
"slices"
"sync"
@ -12,19 +11,17 @@ import (
)
func (p *ForwardProxyCluster) ValidateProxiesThread() {
log.Println("Doing initial backend check, please wait...")
log.Infoln("Doing initial backend check, please wait...")
started := false
// TODO: config value
var sem = semaphore.NewWeighted(int64(50))
var sem = semaphore.NewWeighted(int64(config.GetConfig().MaxProxyCheckers))
ctx := context.TODO()
for {
// TODO: need to have these be temp vars and then copy them over when finished
allProxies := removeDuplicates(append(testProxies, testSmartproxyPool...))
allProxies := removeDuplicates(append(config.GetConfig().ProxyPoolOurs, config.GetConfig().ProxyPoolThirdparty...))
p.ourOnlineProxies = make([]string, 0)
p.smartproxyOnlineProxies = make([]string, 0)
p.smartproxyBrokenProxies = make([]string, 0)
p.thirdpartyOnlineProxies = make([]string, 0)
p.thirdpartyBrokenProxies = make([]string, 0)
p.ipAddresses = make([]string, 0)
var wg sync.WaitGroup
@ -35,62 +32,69 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
defer wg.Done()
if err := sem.Acquire(ctx, 1); err != nil {
fmt.Printf("Failed to acquire semaphore: %v\n", err)
log.Errorf("Validate - failed to acquire semaphore: %v\n", err)
return
}
defer sem.Release(1)
_, _, proxyHost, _, err := splitProxyURL(pxy)
if err != nil {
log.Errorf(`Invalid proxy "%s"`, pxy)
return
}
// Test the proxy.
ipAddr, testErr := sendRequestThroughProxy(pxy, testTargetUrl)
ipAddr, testErr := sendRequestThroughProxy(pxy, config.GetConfig().IpCheckerURL)
if testErr != nil {
fmt.Printf("Proxy %s failed: %s\n", pxy, testErr)
log.Warnf("Validate - proxy %s failed: %s", proxyHost, testErr)
return
}
if slices.Contains(p.ipAddresses, ipAddr) {
fmt.Printf("Duplicate IP Address %s for proxy %s\n", ipAddr, pxy)
log.Warnf("Validate - duplicate IP Address %s for proxy %s", ipAddr, proxyHost)
return
}
p.ipAddresses = append(p.ipAddresses, ipAddr)
// Sort the proxy into the right groups.
if IsSmartproxy(pxy) {
p.smartproxyOnlineProxies = append(p.smartproxyOnlineProxies, pxy)
for _, d := range testSmartproxyBV3HIFix {
if isThirdparty(pxy) {
p.mu.Lock()
p.thirdpartyOnlineProxies = append(p.thirdpartyOnlineProxies, pxy)
p.mu.Unlock()
for _, d := range config.GetConfig().ThirdpartyTestUrls {
_, bv3hiErr := sendRequestThroughProxy(pxy, d)
if bv3hiErr != nil {
fmt.Printf("Smartproxy %s failed: %s\n", pxy, bv3hiErr)
p.smartproxyBrokenProxies = append(p.smartproxyBrokenProxies, pxy)
log.Debugf("Validate - Third-party %s failed: %s\n", proxyHost, bv3hiErr)
p.thirdpartyBrokenProxies = append(p.thirdpartyBrokenProxies, pxy)
}
}
} else {
p.mu.Lock()
p.ourOnlineProxies = append(p.ourOnlineProxies, pxy)
p.mu.Unlock()
}
}(pxy)
}
wg.Wait()
if !started {
p.mu.Lock()
p.ourOnlineProxies = shuffle(p.ourOnlineProxies)
p.smartproxyOnlineProxies = shuffle(p.smartproxyOnlineProxies)
p.thirdpartyOnlineProxies = shuffle(p.thirdpartyOnlineProxies)
p.mu.Unlock()
started = true
p.BalancerOnline.Done()
}
log.Printf("Our Endpoints Online: %d, Smartproxy Endpoints Online: %d, Smartproxy Broken Backends: %d, Total Online: %d\n",
len(p.ourOnlineProxies), len(p.smartproxyOnlineProxies), len(p.smartproxyBrokenProxies), len(p.ourOnlineProxies)+(len(p.smartproxyOnlineProxies)-len(p.smartproxyBrokenProxies)))
p.mu.RLock()
log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d\n",
len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+(len(p.thirdpartyOnlineProxies)-len(p.thirdpartyBrokenProxies)))
p.mu.RUnlock()
time.Sleep(60 * time.Second)
}
}
func getKeysFromMap(m map[string]string) []string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
return keys
}
func shuffle(vals []string) []string {
r := rand.New(rand.NewSource(time.Now().Unix()))
ret := make([]string, len(vals))

25
src/proxy/waitgroup.go Normal file
View File

@ -0,0 +1,25 @@
package proxy
import (
"sync"
"sync/atomic"
)
type WaitGroupCountable struct {
sync.WaitGroup
count int64
}
func (wg *WaitGroupCountable) Add(delta int) {
atomic.AddInt64(&wg.count, int64(delta))
wg.WaitGroup.Add(delta)
}
func (wg *WaitGroupCountable) Done() {
atomic.AddInt64(&wg.count, -1)
wg.WaitGroup.Done()
}
func (wg *WaitGroupCountable) GetCount() int {
return int(atomic.LoadInt64(&wg.count))
}