Merge pull request #973 from projectdiscovery/dev

ehsandeep · web-flow · commit 954cbe6e95a1 · 2023-02-09T23:02:39.000+05:30
httpx v1.0.7
diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml
@@ -32,7 +32,7 @@ jobs:
           password: ${{ secrets.DOCKER_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           platforms: linux/amd64,linux/arm64,linux/arm
diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml
@@ -16,7 +16,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.3.1
+        uses: golangci/golangci-lint-action@v3.4.0
         with:
           version: latest
           args: --timeout 5m
diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,9 @@
-FROM golang:1.19.4-alpine AS builder
+FROM golang:1.20.0-alpine AS builder
 ARG BUILD_SOURCE_TAG=latest
 RUN apk add --no-cache git build-base gcc musl-dev
 RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@${BUILD_SOURCE_TAG}
 
-FROM alpine:3.17.0
+FROM alpine:3.17.1
 RUN apk -U upgrade --no-cache \
     && apk add --no-cache bind-tools ca-certificates
 COPY --from=builder /go/bin/httpx /usr/local/bin/
diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/httpx/runner"
+	errorutil "github.com/projectdiscovery/utils/errors"
 )
 
 func main() {
@@ -58,3 +59,9 @@ func main() {
 	httpxRunner.RunEnumeration()
 	httpxRunner.Close()
 }
+
+func init() {
+	if os.Getenv("DEBUG") != "" {
+		errorutil.ShowStackTrace = true
+	}
+}
diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go
@@ -44,7 +44,8 @@ func (h *httpxLibrary) Execute() error {
 
 	httpxRunner.RunEnumeration()
 
-	expected := "https://scanme.sh:443"
+	// httpx removes default ports for simplicity Ref: https://pkg.go.dev/github.com/projectdiscovery/httpx/common/stringz#RemoveURLDefaultPort
+	expected := "https://scanme.sh"
 
 	if got != expected {
 		return errIncorrectResult(expected, got)
diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go
@@ -19,6 +19,7 @@ import (
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
 	pdhttputil "github.com/projectdiscovery/utils/http"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	urlutil "github.com/projectdiscovery/utils/url"
 	"golang.org/x/net/context"
 	"golang.org/x/net/http2"
 )
@@ -330,11 +331,15 @@ func (h *HTTPX) NewRequest(method, targetURL string) (req *retryablehttp.Request
 
 // NewRequest from url
 func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL string) (req *retryablehttp.Request, err error) {
-	req, err = retryablehttp.NewRequestWithContext(ctx, method, targetURL, nil)
+	urlx, err := urlutil.ParseURL(targetURL, h.Options.Unsafe)
 	if err != nil {
-		return
+		return nil, err
 	}
 
+	req, err = retryablehttp.NewRequestFromURLWithContext(ctx, method, urlx, nil)
+	if err != nil {
+		return nil, err
+	}
 	// Skip if unsafe is used
 	if !h.Options.Unsafe {
 		// set default user agent
diff --git a/common/stringz/stringz.go b/common/stringz/stringz.go
@@ -93,19 +93,18 @@ func RemoveURLDefaultPort(rawURL string) string {
 		return rawURL
 	}
 
-	if u.Scheme == urlutil.HTTP && u.Port == "80" || u.Scheme == urlutil.HTTPS && u.Port == "443" {
-		u.Port = ""
+	if u.Scheme == urlutil.HTTP && u.Port() == "80" || u.Scheme == urlutil.HTTPS && u.Port() == "443" {
+		u.TrimPort()
 	}
 	return u.String()
 }
 
 func GetInvalidURI(rawURL string) (bool, string) {
 	if _, err := url.Parse(rawURL); err != nil {
 		if u, err := urlutil.Parse(rawURL); err == nil {
-			return true, u.RequestURI
+			return true, u.GetRelativePath()
 		}
 	}
-
 	return false, ""
 }
 
diff --git a/go.mod b/go.mod
@@ -10,31 +10,29 @@ require (
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/microcosm-cc/bluemonday v1.0.21
+	github.com/microcosm-cc/bluemonday v1.0.22
 	github.com/miekg/dns v1.1.50 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1
-	github.com/projectdiscovery/clistats v0.0.9
+	github.com/projectdiscovery/clistats v0.0.12
 	github.com/projectdiscovery/fdmax v0.0.4
-	github.com/projectdiscovery/fileutil v0.0.3 // indirect
 	github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c
 	github.com/projectdiscovery/goflags v0.1.6
 	github.com/projectdiscovery/gologger v1.1.7
-	github.com/projectdiscovery/hmap v0.0.6
+	github.com/projectdiscovery/hmap v0.0.7
 	github.com/projectdiscovery/iputil v0.0.2 // indirect
 	github.com/projectdiscovery/mapcidr v1.0.3
-	github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e
-	github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d
-	github.com/projectdiscovery/sliceutil v0.0.1 // indirect
+	github.com/projectdiscovery/rawhttp v0.1.9-0.20230208140210-81c468e769ed
+	github.com/projectdiscovery/retryablehttp-go v1.0.11-0.20230208144003-54f3176b2916
 	github.com/projectdiscovery/stringsutil v0.0.2 // indirect
-	github.com/projectdiscovery/wappalyzergo v0.0.76
+	github.com/projectdiscovery/wappalyzergo v0.0.81
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.4.0
-	go.etcd.io/bbolt v1.3.6 // indirect
+	go.etcd.io/bbolt v1.3.7 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
-	golang.org/x/net v0.5.0
-	golang.org/x/sys v0.4.0 // indirect
-	golang.org/x/text v0.6.0
+	golang.org/x/net v0.6.0
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0
 )
 
 require github.com/spaolacci/murmur3 v1.1.0
@@ -47,13 +45,13 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/projectdiscovery/asnmap v0.0.1
 	github.com/projectdiscovery/dsl v0.0.3
-	github.com/projectdiscovery/fastdialer v0.0.19
-	github.com/projectdiscovery/ratelimit v0.0.4
-	github.com/projectdiscovery/tlsx v1.0.3
-	github.com/projectdiscovery/utils v0.0.4-0.20230102120019-c7a04e2045be
+	github.com/projectdiscovery/fastdialer v0.0.22
+	github.com/projectdiscovery/ratelimit v0.0.6
+	github.com/projectdiscovery/tlsx v1.0.4
+	github.com/projectdiscovery/utils v0.0.8-0.20230208160155-47c6b52af9f0
 	github.com/stretchr/testify v1.8.1
 	go.uber.org/multierr v1.9.0
-	golang.org/x/exp v0.0.0-20221230185412-738e83a70c30
+	golang.org/x/exp v0.0.0-20230206171751-46f607a40771
 )
 
 require (
@@ -66,58 +64,60 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect
-	github.com/cockroachdb/errors v1.9.0 // indirect
-	github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
-	github.com/cockroachdb/pebble v0.0.0-20221229212011-811a8c0e741b // indirect
+	github.com/cockroachdb/errors v1.9.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230207164304-7d1e4ba7ffd0 // indirect
 	github.com/cockroachdb/redact v1.1.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dsnet/compress v0.0.1 // indirect
-	github.com/getsentry/sentry-go v0.16.0 // indirect
+	github.com/getsentry/sentry-go v0.18.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.15.13 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mholt/archiver v3.1.1+incompatible // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nwaples/rardecode v1.1.0 // indirect
+	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/pierrec/lz4 v2.6.0+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect
+	github.com/projectdiscovery/freeport v0.0.4 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.3 // indirect
 	github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect
-	github.com/projectdiscovery/retryabledns v1.0.17 // indirect
+	github.com/projectdiscovery/retryabledns v1.0.20 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.39.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
-	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
-	github.com/tidwall/btree v1.4.3 // indirect
+	github.com/tidwall/btree v1.6.0 // indirect
 	github.com/tidwall/buntdb v1.2.10 // indirect
-	github.com/tidwall/gjson v1.14.3 // indirect
+	github.com/tidwall/gjson v1.14.4 // indirect
 	github.com/tidwall/grect v0.1.4 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
-	github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 // indirect
+	github.com/weppos/publicsuffix-go v0.20.0 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yl2chen/cidranger v1.0.2 // indirect
-	github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect
+	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
 	github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4 // indirect
-	github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be // indirect
-	golang.org/x/crypto v0.1.0 // indirect
-	golang.org/x/mod v0.6.0 // indirect
-	golang.org/x/tools v0.2.0 // indirect
+	github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 // indirect
+	golang.org/x/crypto v0.5.0 // indirect
+	golang.org/x/mod v0.8.0 // indirect
+	golang.org/x/tools v0.5.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/djherbis/times.v1 v1.3.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
diff --git a/go.sum b/go.sum
diff --git a/runner/banner.go b/runner/banner.go
diff --git a/runner/runner.go b/runner/runner.go

Original file line number	Diff line number	Diff line change
`@@ -93,19 +93,18 @@ func RemoveURLDefaultPort(rawURL string) string {`
`93`	`93`	`return rawURL`
`94`	`94`	`}`
`95`	`95`
`96`		`- if u.Scheme == urlutil.HTTP && u.Port == "80" \|\| u.Scheme == urlutil.HTTPS && u.Port == "443" {`
`97`		`- u.Port = ""`
	`96`	`+ if u.Scheme == urlutil.HTTP && u.Port() == "80" \|\| u.Scheme == urlutil.HTTPS && u.Port() == "443" {`
	`97`	`+ u.TrimPort()`
`98`	`98`	`}`
`99`	`99`	`return u.String()`
`100`	`100`	`}`
`101`	`101`
`102`	`102`	`func GetInvalidURI(rawURL string) (bool, string) {`
`103`	`103`	`if _, err := url.Parse(rawURL); err != nil {`
`104`	`104`	`if u, err := urlutil.Parse(rawURL); err == nil {`
`105`		`- return true, u.RequestURI`
	`105`	`+ return true, u.GetRelativePath()`
`106`	`106`	`}`
`107`	`107`	`}`
`108`		`-`
`109`	`108`	`return false, ""`
`110`	`109`	`}`
`111`	`110`