Merge pull request #288 from projectdiscovery/dev

Minor bugfix

Author: ehsandeep

common/stringz/stringz.go

@@ -8,11 +8,13 @@ import (
 )
 
 // TrimProtocol removes the HTTP scheme from an URI
-func TrimProtocol(targetURL string) string {
+func TrimProtocol(targetURL string, addDefaultPort bool) string {
 	URL := strings.TrimSpace(targetURL)
 	if strings.HasPrefix(strings.ToLower(URL), "http://") || strings.HasPrefix(strings.ToLower(URL), "https://") {
-		URL = AddURLDefaultPort(URL)
-		URL = URL[strings.Index(URL, "//")+2:]
+		if addDefaultPort {
+			URL = AddURLDefaultPort(URL)
+			URL = URL[strings.Index(URL, "//")+2:]
+		}
 	}
 
 	return URL

runner/banner.go

@@ -8,11 +8,11 @@ const banner = `
   / __ \/ __/ __/ __ \|   /
  / / / / /_/ /_/ /_/ /   |
 /_/ /_/\__/\__/ .___/_/|_|
-             /_/              v1.0.7
+             /_/              v1.0.8
 `
 
 // Version is the current version of httpx
-const Version = `v1.0.7`
+const Version = `v1.0.8`
 
 // showBanner is used to show the banner to the user
 func showBanner() {

runner/options.go

@@ -51,6 +51,7 @@ type scanOptions struct {
 	OutputResponseTime     bool
 	PreferHTTPS            bool
 	NoFallback             bool
+	NoFallbackScheme       bool
 	TechDetect             bool
 	StoreChain             bool
 	MaxResponseBodySize    int
@@ -88,6 +89,7 @@ func (s *scanOptions) Clone() *scanOptions {
 		OutputResponseTime:     s.OutputResponseTime,
 		PreferHTTPS:            s.PreferHTTPS,
 		NoFallback:             s.NoFallback,
+		NoFallbackScheme:       s.NoFallbackScheme,
 		TechDetect:             s.TechDetect,
 		StoreChain:             s.StoreChain,
 		OutputExtractRegex:     s.OutputExtractRegex,
@@ -159,6 +161,7 @@ type Options struct {
 	OutputCDN                 bool
 	OutputResponseTime        bool
 	NoFallback                bool
+	NoFallbackScheme          bool
 	TechDetect                bool
 	TLSGrab                   bool
 	protocol                  string
@@ -231,6 +234,7 @@ func ParseOptions() *Options {
 	flag.BoolVar(&options.OutputCDN, "cdn", false, "Check if domain's ip belongs to known CDN (akamai, cloudflare, ..)")
 	flag.BoolVar(&options.OutputResponseTime, "response-time", false, "Output the response time")
 	flag.BoolVar(&options.NoFallback, "no-fallback", false, "If HTTPS on port 443 is successful on default configuration, probes also port 80 for HTTP")
+	flag.BoolVar(&options.NoFallbackScheme, "no-fallback-scheme", false, "The tool will respect and attempt the scheme specified in the url (if HTTPS is specified no HTTP is attempted)")
 	flag.BoolVar(&options.ShowStatistics, "stats", false, "Enable statistic on keypress (terminal may become unresponsive till the end)")
 	flag.BoolVar(&options.RandomAgent, "random-agent", false, "Use randomly selected HTTP User-Agent header value")
 	flag.BoolVar(&options.StoreChain, "store-chain", false, "Save chain to file (default 'output')")

runner/runner.go

@@ -10,6 +10,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httputil"
+	"net/url"
 	"os"
 	"path"
 	"regexp"
@@ -177,6 +178,7 @@ func New(options *Options) (*Runner, error) {
 	scanopts.OutputCDN = options.OutputCDN
 	scanopts.OutputResponseTime = options.OutputResponseTime
 	scanopts.NoFallback = options.NoFallback
+	scanopts.NoFallbackScheme = options.NoFallbackScheme
 	scanopts.TechDetect = options.TechDetect
 	scanopts.StoreChain = options.StoreChain
 	scanopts.MaxResponseBodySize = options.MaxResponseBodySize
@@ -414,8 +416,10 @@ func (r *Runner) RunEnumeration() {
 		var reqs int
 		protocol := r.options.protocol
 		// attempt to parse url as is
-		if u, err := urlutil.Parse(t); err == nil {
-			protocol = u.Scheme
+		if u, err := url.Parse(t); err == nil {
+			if r.options.NoFallbackScheme && u.Scheme == httpx.HTTP || u.Scheme == httpx.HTTPS {
+				protocol = u.Scheme
+			}
 		}
 
 		if len(r.options.requestURIs) > 0 {
@@ -448,7 +452,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.
 	if scanopts.NoFallback {
 		protocols = []string{httpx.HTTPS, httpx.HTTP}
 	}
-	for target := range targets(stringz.TrimProtocol(t)) {
+	for target := range targets(stringz.TrimProtocol(t, scanopts.NoFallback || scanopts.NoFallbackScheme)) {
 		// if no custom ports specified then test the default ones
 		if len(customport.Ports) == 0 {
 			for _, method := range scanopts.Methods {
@@ -550,10 +554,13 @@ retry:
 	URL, _ := urlutil.Parse(domain)
 	URL.Scheme = protocol
 
+	if !strings.Contains(domain, URL.Port) {
+		URL.Port = ""
+	}
+
 	if !scanopts.Unsafe {
 		URL.RequestURI += scanopts.RequestURI
 	}
-
 	req, err := hp.NewRequest(method, URL.String())
 	if err != nil {
 		return Result{URL: URL.String(), err: err}