Merge pull request #5 from kangoo13/feat/module-new-checkers#4

feat: Modulation of Repository to Let New Checkers

Author: kangoo13

.gitignore

@@ -15,7 +15,7 @@
 # vendor/
 
 proxies.txt
-cloudflare-proxy-ban-checker
+proxy-checker
 bad.txt
 good.txt
 .idea

cfprxchecker/checker.go

@@ -0,0 +1,44 @@
+package checkers
+
+import (
+	"github.com/gocolly/colly"
+	"log"
+	"net/url"
+)
+
+func CheckIsAlive(args *Wrapper) {
+	var onErrorCallback = func(response *colly.Response, err error) {
+		log.Printf("[DEBUG CheckIsAlive] Bad proxy found error status %d [%s] [%s]", response.StatusCode, response.Request.ProxyURL, err)
+	}
+
+	var onResponseCallback = func(response *colly.Response) {
+		log.Printf("[DEBUG CheckIsAlive] Good proxy found [%s]", response.Request.ProxyURL)
+		args.AddGoodProxy(response.Request.ProxyURL)
+	}
+
+	var collectors []*colly.Collector
+
+	for _, proxy := range args.ProxiesToTest {
+		newCollector := NewCollector(args)
+		newCollector.OnResponse(onResponseCallback)
+		newCollector.OnError(onErrorCallback)
+		u, err := url.Parse(proxy)
+		if err != nil {
+			log.Printf("[CheckIsAlive] error while parseing proxy %s %s", proxy, err)
+			newCollector = nil
+			continue
+		}
+		newCollector.SetProxyFunc(ProxyURL(u))
+		if err = newCollector.Visit(args.WebsiteToCrawl); err != nil {
+			log.Printf("[CheckIsAlive] error happening doing Visit %s", err)
+			newCollector = nil
+			continue
+
+		}
+		collectors = append(collectors, newCollector)
+	}
+
+	for _, collec := range collectors {
+		collec.Wait()
+	}
+}

cfprxchecker/custom_round_robin_proxy.go

@@ -0,0 +1,45 @@
+package checkers
+
+import (
+	"github.com/gocolly/colly"
+	"log"
+	"net/url"
+)
+
+func CheckCloudFlareBypass(args *Wrapper) {
+	var onErrorCallback = func(response *colly.Response, err error) {
+		log.Printf("[DEBUG CheckCloudFlareBypass] Bad proxy found error status %d [%s] [%s]", response.StatusCode, response.Request.ProxyURL, err)
+	}
+
+	var onResponseCallback = func(response *colly.Response) {
+		log.Printf("[DEBUG CheckCloudFlareBypass] Good proxy found [%s]", response.Request.ProxyURL)
+		args.AddGoodProxy(response.Request.ProxyURL)
+	}
+
+	var collectors []*colly.Collector
+
+	for _, proxy := range args.ProxiesToTest {
+		newCollector := NewCollector(args)
+		newCollector.OnResponse(onResponseCallback)
+		newCollector.OnError(onErrorCallback)
+		u, err := url.Parse(proxy)
+		if err != nil {
+			log.Printf("[CheckCloudFlareBypass] error while parseing proxy %s %s", proxy, err)
+			newCollector = nil
+			continue
+		}
+		newCollector.SetProxyFunc(ProxyURL(u))
+		if err = newCollector.Visit(args.WebsiteToCrawl); err != nil {
+			log.Printf("[CheckCloudFlareBypass] error happening doing Visit %s", err)
+			newCollector = nil
+			continue
+
+		}
+		collectors = append(collectors, newCollector)
+	}
+
+	for _, collec := range collectors {
+		collec.Wait()
+	}
+
+}

checkers/alive.go

@@ -0,0 +1,31 @@
+package checkers
+
+import (
+	"sync"
+	"time"
+)
+
+type Wrapper struct {
+	WebsiteToCrawl  string
+	ProxiesToTest   []string
+	AcceptedProxies []string
+	TimeoutProxy    time.Duration
+	mu              sync.Mutex
+	checks          int
+}
+
+func (a *Wrapper) AddGoodProxy(proxy string) {
+	a.mu.Lock()
+
+	a.AcceptedProxies = append(a.AcceptedProxies, proxy)
+
+	a.mu.Unlock()
+}
+
+func (a *Wrapper) PrepareNextChecker() {
+	if a.checks != 0 {
+		a.ProxiesToTest = a.AcceptedProxies
+		a.AcceptedProxies = []string{}
+	}
+	a.checks += 1
+}

checkers/cloudflare_bypass.go

@@ -0,0 +1,42 @@
+package checkers
+
+import (
+	"context"
+	"github.com/gocolly/colly"
+	"github.com/gocolly/colly/extensions"
+	"net/http"
+	"net/url"
+)
+
+// ProxyURL returns a proxy function (for use in a Transport)
+// that always returns the same URL.
+func ProxyURL(fixedURL *url.URL) func(*http.Request) (*url.URL, error) {
+	return func(pr *http.Request) (*url.URL, error) {
+		ctx := context.WithValue(pr.Context(), colly.ProxyURLKey, fixedURL.String())
+		*pr = *pr.WithContext(ctx)
+		return fixedURL, nil
+	}
+}
+
+func NewCollector(args *Wrapper) *colly.Collector {
+	// Instantiate default collector
+	c := colly.NewCollector(
+		colly.Async(true),
+	)
+
+	c.IgnoreRobotsTxt = true
+	c.AllowURLRevisit = true
+	c.CacheDir = ""
+
+	c.WithTransport(&http.Transport{
+		DisableKeepAlives:   true,
+		MaxIdleConns:        100,
+		MaxIdleConnsPerHost: 100,
+	})
+
+	c.SetRequestTimeout(args.TimeoutProxy)
+
+	extensions.RandomUserAgent(c)
+
+	return c
+}

checkers/models.go

@@ -0,0 +1,27 @@
+package core
+
+import (
+	"github.com/kangoo13/proxy-checker/checkers"
+)
+
+func CheckProxies(proxyChecker *ProxyChecker) []string {
+	var checkersWrapper = checkers.Wrapper{
+		ProxiesToTest:   proxyChecker.ProxyList,
+		AcceptedProxies: []string{},
+		TimeoutProxy:    proxyChecker.TimeoutProxy,
+	}
+
+	if proxyChecker.Basic {
+		checkersWrapper.WebsiteToCrawl = "https://httpbin.org/get"
+		checkersWrapper.PrepareNextChecker()
+		checkers.CheckIsAlive(&checkersWrapper)
+	}
+
+	if proxyChecker.CloudFlareBypass {
+		checkersWrapper.WebsiteToCrawl = proxyChecker.WebsiteCloudFlare
+		checkersWrapper.PrepareNextChecker()
+		checkers.CheckCloudFlareBypass(&checkersWrapper)
+	}
+
+	return checkersWrapper.AcceptedProxies
+}