Merge pull request #928 from projectdiscovery/dev

httpx v1.2.6

Author: ehsandeep

.github/workflows/build-test.yml

@@ -1,7 +1,6 @@
 name: 🔨 Build Test
 
 on:
-  push:
   pull_request:
   workflow_dispatch:
 
@@ -16,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
 
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/codeql-analysis.yml

@@ -2,7 +2,6 @@ name: 🚨 CodeQL Analysis
 
 on:
   workflow_dispatch:
-  push:
   pull_request:
     branches:
       - dev

.github/workflows/functional-test.yml

@@ -1,6 +1,6 @@
 name: 🧪 Functional Test
+
 on:
-  push:
   pull_request:
   workflow_dispatch:
 
@@ -15,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
 
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/lint-test.yml

@@ -1,6 +1,6 @@
 name: 🙏🏻 Lint Test
+
 on:
-  push:
   pull_request:
   workflow_dispatch:
 
@@ -12,11 +12,11 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.18
+          go-version: 1.19
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.3.0
+        uses: golangci/golangci-lint-action@v3.3.1
         with:
           version: latest
           args: --timeout 5m

.github/workflows/release-binary.yml

@@ -1,8 +1,9 @@
 name: 🎉 Release Binary
+
 on:
-  create:
+  push:
     tags:
-      - v*
+      - '*'
   workflow_dispatch:
 
 jobs:
@@ -17,14 +18,16 @@ jobs:
       - name: "Set up Go"
         uses: actions/setup-go@v3
         with: 
-          go-version: 1.18
+          go-version: 1.19
 
       - name: "Create release on GitHub"
-        uses: goreleaser/goreleaser-action@v3
+        uses: goreleaser/goreleaser-action@v4
         with: 
           args: "release --rm-dist"
           version: latest
           workdir: .
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}"
+          DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}"
+          DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}"

.github/workflows/sonarcloud.yml

@@ -1,11 +1,7 @@
 name: 👮🏼‍♂️ Sonarcloud
+
 on:
-  push:
-    branches:
-      - master
-      - dev
   pull_request:
-    types: [opened, synchronize, reopened]
   workflow_dispatch:
 
 jobs:
@@ -20,7 +16,7 @@ jobs:
       - name: "Set up Go"
         uses: actions/setup-go@v3
         with: 
-          go-version: 1.18
+          go-version: 1.19
 
       - name: Run unit Tests
         run: |

.goreleaser.yml

@@ -39,4 +39,8 @@ announce:
     enabled: true
     channel: '#release'
     username: GoReleaser
-    message_template: '{{ .ProjectName }} {{ .Tag }} is out! Check it out at {{ .ReleaseURL }}'
+    message_template: 'New Release: {{ .ProjectName }} {{ .Tag }} is published! Check it out at {{ .ReleaseURL }}'
+
+  discord:
+    enabled: true
+    message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}'

Dockerfile

@@ -1,8 +1,9 @@
-FROM golang:1.19.2-alpine AS builder
-RUN apk add --no-cache git
-RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest
+FROM golang:1.19.4-alpine AS builder
+ARG BUILD_SOURCE_TAG=latest
+RUN apk add --no-cache git build-base gcc musl-dev
+RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@${BUILD_SOURCE_TAG}
 
-FROM alpine:3.16.2
+FROM alpine:3.17.0
 RUN apk -U upgrade --no-cache \
     && apk add --no-cache bind-tools ca-certificates
 COPY --from=builder /go/bin/httpx /usr/local/bin/

README.md

@@ -1,5 +1,5 @@
 <h1 align="center">
-  <img src="static/httpx-logo.png" alt="httpx" width="200px"></a>
+  <img src="static/httpx-logo.png" alt="httpx" width="200px">
   <br>
 </h1>
 
@@ -24,12 +24,12 @@
 </p>
 
 
-httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the [retryablehttp](https://github.com/projectdiscovery/retryablehttp-go) library. It is designed to maintain result reliability with an increased number of threads.
+`httpx` is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the [retryablehttp](https://github.com/projectdiscovery/retryablehttp-go) library. It is designed to maintain result reliability with an increased number of threads.
 
 # Features
 
 <h1 align="center">
-  <img src="https://user-images.githubusercontent.com/8293321/135731750-4c1d38b1-bd2a-40f9-88e9-3c4b9f6da378.png" alt="httpx" width="700px"></a>
+  <img src="https://user-images.githubusercontent.com/8293321/135731750-4c1d38b1-bd2a-40f9-88e9-3c4b9f6da378.png" alt="httpx" width="700px">
   <br>
 </h1>
 
@@ -40,7 +40,7 @@ httpx is a fast and multi-purpose HTTP toolkit that allows running multiple prob
  - Supports hosts, URLs and CIDR as input.
  - Handles edge cases doing retries, backoffs etc for handling WAFs.
 
-### Supported probes:-
+### Supported probes
 
 | Probes          | Default check | Probes         | Default check |
 |-----------------|---------------|----------------|---------------|
@@ -62,7 +62,7 @@ httpx is a fast and multi-purpose HTTP toolkit that allows running multiple prob
 
 # Installation Instructions
 
-httpx requires **go1.17** to install successfully. Run the following command to get the repo - 
+`httpx` requires **go1.19** to install successfully. Run the following command to get the repo:
 
 ```sh
 go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest
@@ -117,7 +117,7 @@ MATCHERS:
    -mfc, -match-favicon string[]      match response with specified favicon hash (-mfc 1494302000)
    -ms, -match-string string          match response with specified string (-ms admin)
    -mr, -match-regex string           match response with specified regex (-mr admin)
-   -mcdn, -match-cdn string[]         match host with specified cdn provider (fastly, incapsula, oracle, google, azure, cloudflare, cloudfront, sucuri, leaseweb, akamai)
+   -mcdn, -match-cdn string[]         match host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri)
    -mrt, -match-response-time string  match response with specified response time in seconds (-mrt '< 1')
    -mdc, -match-condition string      match response with dsl expression condition
 
@@ -133,7 +133,7 @@ FILTERS:
    -ffc, -filter-favicon string[]      filter response with specified favicon hash (-mfc 1494302000)
    -fs, -filter-string string          filter response with specified string (-fs admin)
    -fe, -filter-regex string           filter response with specified regex (-fe admin)
-   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (fastly, incapsula, oracle, google, azure, cloudflare, cloudfront, sucuri, leaseweb, akamai)
+   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri)
    -frt, -filter-response-time string  filter response with specified response time in seconds (-frt '> 1')
    -fdc, -filter-condition string      filter response with dsl expression condition
 
@@ -155,14 +155,16 @@ MISCELLANEOUS:
    -ldv, -list-dsl-variables  list json output field keys name that support dsl matcher/filter
 
 OUTPUT:
-   -o, -output string                file to write output results
-   -sr, -store-response              store http response to output directory
-   -srd, -store-response-dir string  store http response to custom directory
-   -csv                              store output in csv format
-   -json                             store output in JSONL(ines) format
-   -irr, -include-response           include http request/response in JSON output (-json only)
-   -include-chain                    include redirect http chain in JSON output (-json only)
-   -store-chain                      include http redirect chain in responses (-sr only)
+   -o, -output string                  file to write output results
+   -sr, -store-response                store http response to output directory
+   -srd, -store-response-dir string    store http response to custom directory
+   -csv                                store output in csv format
+   -csvo, -csv-output-encoding string  define output encoding
+   -json                               store output in JSONL(ines) format
+   -irr, -include-response             include http request/response in JSON output (-json only)
+   -irrb, -include-response-base64     include base64 encoded http request/response in JSON output (-json only)
+   -include-chain                      include redirect http chain in JSON output (-json only)
+   -store-chain                        include http redirect chain in responses (-sr only)
 
 CONFIGURATIONS:
    -r, -resolvers string[]       list of custom resolver (file or comma separated)
@@ -183,6 +185,7 @@ CONFIGURATIONS:
    -s, -stream                   stream mode - start elaborating input targets without sorting
    -sd, -skip-dedupe             disable dedupe input items (only used with stream mode)
    -ldp, -leave-default-ports    leave default http/https ports in host header (eg. http://host:80 - https//host:443
+   -ztls                         use ztls library with autofallback to standard one for tls13
 
 DEBUG:
    -health-check, -hc        run diagnostic check up
@@ -204,6 +207,7 @@ OPTIMIZATIONS:
    -ec, -exclude-cdn                  skip full port scans for CDNs (only checks for 80,443)
    -retries int                       number of retries
    -timeout int                       timeout in seconds (default 5)
+   -delay duration                    duration between each http request (eg: 200ms, 1s) (default -1ns)
    -rsts, -response-size-to-save int  max response size to save in bytes (default 2147483647)
    -rstr, -response-size-to-read int  max response size to read in bytes (default 2147483647)
 ```
@@ -241,7 +245,7 @@ https://support.hackerone.com
 
 ### File Input
 
-This will run the tool with the `probe` flag against all of the hosts in **hosts.txt** and return URLs with probed status.
+This will run the tool with the `-probe` flag against all the hosts in **hosts.txt** and return URLs with probed status.
 
 ```console
 httpx -list hosts.txt -silent -probe
@@ -467,31 +471,30 @@ https://docs.hackerone.com
 https://support.hackerone.com
 ```
 
-### Using httpx as a library
-`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to close at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it:
+### Using `httpx` as a library
+`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it:
 
 ```go
 package main
 
 import (
 	"log"
-	"os"
 
+	"github.com/projectdiscovery/goflags"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/gologger/levels"
 	"github.com/projectdiscovery/httpx/runner"
 )
 
 func main() {
-	inputFile := "test.txt"
-	err := os.WriteFile(inputFile, []byte("scanme.sh"), 0644)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer os.RemoveAll(inputFile)
+	gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional)
 
 	options := runner.Options{
-		Methods:   "GET",
-		InputFile: inputFile,
+		Methods: "GET",
+		InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io"},
+		//InputFile: "./targetDomains.txt", // path to file containing the target domains list 
 	}
+
 	if err := options.ValidateOptions(); err != nil {
 		log.Fatal(err)
 	}
@@ -509,17 +512,27 @@ func main() {
 
 # Notes
 
-- As default, **httpx** checks for `HTTPS` probe and fall-back to `HTTP` only if `HTTPS` is not reachable.
-- For printing both HTTP/HTTPS results, `no-fallback` flag can be used.
+- As default, `httpx` checks for **HTTPS** probe and fall-back to **HTTP** only if **HTTPS** is not reachable.
+- The `-no-fallback` flag can be used to display both **HTTP** and **HTTPS** results 
 - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
-- `favicon`,`vhost`, `http2`, `pipeline`, `ports`, `csp-probe`, `tls-probe` and `path` are unique flag with different probes.
-- Unique flags should be used for specific use cases instead of running them as default with other probes.
-- When using `json` flag, all the information (default probes) included in the JSON output.
-- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port`  (eg **udp:127.0.0.1:53**)
+- The following flags should be used for specific use cases instead of running them as default with other probes:
+  * `-favicon`,`-vhost`, `-http2`, `-pipeline`, `-ports`, `-csp-probe`, `-tls-probe`, `-path`
+- When using the `-json` flag, all the default probe results are included in the JSON output.
+- Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)
 - Invalid custom resolvers/files are ignored.
 
 # Acknowledgement
 
-httpx is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/httpx/blob/master/THANKS.md)** file for more details. Do also check out these similar awesome projects that may fit in your workflow:
+Probing feature is inspired by [@tomnomnom/httprobe](https://github.com/tomnomnom/httprobe) work ❤️
+
+
+--------
+
+<div align="center">
+
+`httpx` is made with 💙 by the [projectdiscovery](https://projectdiscovery.io) team and distributed under [MIT License](LICENSE.md).
+
+
+<a href="https://discord.gg/projectdiscovery"><img src="https://raw.githubusercontent.com/projectdiscovery/nuclei-burp-plugin/main/static/join-discord.png" width="300" alt="Join Discord"></a>
 
-Probing feature is inspired by [@tomnomnom/httprobe](https://github.com/tomnomnom/httprobe) work :heart:
+</div>

cmd/functional-test/main.go

@@ -22,7 +22,7 @@ var (
 
 	mainHttpxBinary = flag.String("main", "", "Main Branch Httpx Binary")
 	devHttpxBinary  = flag.String("dev", "", "Dev Branch Httpx Binary")
-	testcases      = flag.String("testcases", "", "Test cases file for Httpx functional tests")
+	testcases       = flag.String("testcases", "", "Test cases file for Httpx functional tests")
 )
 
 func main() {

cmd/functional-test/testcases.txt

@@ -14,4 +14,6 @@ scanme.sh {{binary}} -silent -unsafe
 scanme.sh {{binary}} -silent -x all
 scanme.sh {{binary}} -silent -body 'a=b'
 scanme.sh {{binary}} -silent -exclude-cdn
-scanme.sh {{binary}} -silent -ports https:443
+scanme.sh {{binary}} -silent -ports https:443
+scanme.sh {{binary}} -silent -ztls
+https://scanme.sh?a=1*1 {{binary}} -silent

cmd/integration-test/http.go

@@ -83,7 +83,7 @@ func (h *standardHttpGet) Execute() error {
 	}
 
 	if h.expectedOutput != "" && !strings.EqualFold(results[0], h.expectedOutput) {
-		return errIncorrectResult(results[0], h.expectedOutput)
+		return errIncorrectResult(h.expectedOutput, results[0])
 	}
 
 	return nil
@@ -115,7 +115,7 @@ func (h *issue276) Execute() error {
 	// title
 	expected := ts.URL + "/redirect" + " [302] [Object moved]"
 	if !strings.EqualFold(results[0], expected) {
-		return errIncorrectResult(results[0], expected)
+		return errIncorrectResult(expected, results[0])
 	}
 	return nil
 }
@@ -146,7 +146,7 @@ func (h *issue277) Execute() error {
 	// title
 	expected := ts.URL + uripath
 	if !strings.EqualFold(results[0], expected) {
-		return errIncorrectResult(results[0], expected)
+		return errIncorrectResult(expected, results[0])
 	}
 	return nil
 }
@@ -175,7 +175,7 @@ func (h *issue303) Execute() error {
 	// full url with port
 	expected := ts.URL
 	if !strings.EqualFold(results[0], expected) {
-		return errIncorrectResult(results[0], expected)
+		return errIncorrectResult(expected, results[0])
 	}
 	return nil
 }
@@ -249,7 +249,7 @@ func (h *issue414) Execute() error {
 	}
 	expected := ts.URL + uripath
 	if !strings.EqualFold(results[0], expected) {
-		return errIncorrectResult(results[0], expected)
+		return errIncorrectResult(expected, results[0])
 	}
 	return nil
 }