Skip to content

Commit ffadd65

Browse files
ehsandeepehsandeep
authored
Merge pull request #80 from projectdiscovery/feature-pipeline
adding http pipeline check
2 parents e2e5501 + 0cd6bd0 commit ffadd65

File tree

4 files changed

+153
-71
lines changed

4 files changed

+153
-71
lines changed

‎cmd/httpx/httpx.go

Lines changed: 82 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -114,93 +114,94 @@ func main() {
114114
scanopts.OutputContentType = options.OutputContentType
115115
scanopts.RequestBody = options.RequestBody
116116
scanopts.Unsafe = options.Unsafe
117+
scanopts.Pipeline = options.Pipeline
117118
scanopts.HTTP2Probe = options.HTTP2Probe
118119
scanopts.OutputMethod = options.OutputMethod
119120
if len(scanopts.Methods) > 0 {
120121
scanopts.OutputMethod = true
121-
}
122-
123-
// Try to create output folder if it doesnt exist
124-
if options.StoreResponse && !fileutil.FolderExists(options.StoreResponseDir) {
125-
if err := os.MkdirAll(options.StoreResponseDir, os.ModePerm); err != nil {
126-
gologger.Fatalf("Could not create output directory '%s': %s\n", options.StoreResponseDir, err)
127-
}
128-
}
129-
130-
// output routine
131-
wgoutput := sizedwaitgroup.New(1)
132-
wgoutput.Add()
133-
output := make(chan Result)
134-
go func(output chan Result) {
135-
defer wgoutput.Done()
136122

137-
var f *os.File
138-
if options.Output != "" {
139-
var err error
140-
f, err = os.Create(options.Output)
141-
if err != nil {
142-
gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, err)
123+
// Try to create output folder if it doesnt exist
124+
if options.StoreResponse && !fileutil.FolderExists(options.StoreResponseDir) {
125+
if err := os.MkdirAll(options.StoreResponseDir, os.ModePerm); err != nil {
126+
gologger.Fatalf("Could not create output directory '%s': %s\n", options.StoreResponseDir, err)
143127
}
144-
defer f.Close()
145128
}
146-
for r := range output {
147-
if r.err != nil {
148-
gologger.Debugf("Failure '%s': %s\n", r.URL, r.err)
149-
continue
150-
}
151129

152-
// apply matchers and filters
153-
if len(options.filterStatusCode) > 0 && slice.IntSliceContains(options.filterStatusCode, r.StatusCode) {
154-
continue
155-
}
156-
if len(options.filterContentLength) > 0 && slice.IntSliceContains(options.filterContentLength, r.ContentLength) {
157-
continue
158-
}
159-
if len(options.matchStatusCode) > 0 && !slice.IntSliceContains(options.matchStatusCode, r.StatusCode) {
160-
continue
130+
// output routine
131+
wgoutput := sizedwaitgroup.New(1)
132+
wgoutput.Add()
133+
output := make(chan Result)
134+
go func(output chan Result) {
135+
defer wgoutput.Done()
136+
137+
var f *os.File
138+
if options.Output != "" {
139+
var err error
140+
f, err = os.Create(options.Output)
141+
if err != nil {
142+
gologger.Fatalf("Could not create output file '%s': %s\n", options.Output, err)
143+
}
144+
defer f.Close()
161145
}
162-
if len(options.matchContentLength) > 0 && !slice.IntSliceContains(options.matchContentLength, r.ContentLength) {
163-
continue
146+
for r := range output {
147+
if r.err != nil {
148+
gologger.Debugf("Failure '%s': %s\n", r.URL, r.err)
149+
continue
150+
}
151+
152+
// apply matchers and filters
153+
if len(options.filterStatusCode) > 0 && slice.IntSliceContains(options.filterStatusCode, r.StatusCode) {
154+
continue
155+
}
156+
if len(options.filterContentLength) > 0 && slice.IntSliceContains(options.filterContentLength, r.ContentLength) {
157+
continue
158+
}
159+
if len(options.matchStatusCode) > 0 && !slice.IntSliceContains(options.matchStatusCode, r.StatusCode) {
160+
continue
161+
}
162+
if len(options.matchContentLength) > 0 && !slice.IntSliceContains(options.matchContentLength, r.ContentLength) {
163+
continue
164+
}
165+
166+
row := r.str
167+
if options.JSONOutput {
168+
row = r.JSON()
169+
}
170+
171+
gologger.Silentf("%s\n", row)
172+
if f != nil {
173+
f.WriteString(row + "\n")
174+
}
164175
}
176+
}(output)
165177

166-
row := r.str
167-
if options.JSONOutput {
168-
row = r.JSON()
169-
}
178+
wg := sizedwaitgroup.New(options.Threads)
179+
var sc *bufio.Scanner
170180

171-
gologger.Silentf("%s\n", row)
172-
if f != nil {
173-
f.WriteString(row + "\n")
181+
// check if file has been provided
182+
if fileutil.FileExists(options.InputFile) {
183+
finput, err := os.Open(options.InputFile)
184+
if err != nil {
185+
gologger.Fatalf("Could read input file '%s': %s\n", options.InputFile, err)
174186
}
187+
defer finput.Close()
188+
sc = bufio.NewScanner(finput)
189+
} else if fileutil.HasStdin() {
190+
sc = bufio.NewScanner(os.Stdin)
191+
} else {
192+
gologger.Fatalf("No input provided")
175193
}
176-
}(output)
177-
178-
wg := sizedwaitgroup.New(options.Threads)
179-
var sc *bufio.Scanner
180194

181-
// check if file has been provided
182-
if fileutil.FileExists(options.InputFile) {
183-
finput, err := os.Open(options.InputFile)
184-
if err != nil {
185-
gologger.Fatalf("Could read input file '%s': %s\n", options.InputFile, err)
195+
for sc.Scan() {
196+
process(sc.Text(), &wg, hp, protocol, scanopts, output)
186197
}
187-
defer finput.Close()
188-
sc = bufio.NewScanner(finput)
189-
} else if fileutil.HasStdin() {
190-
sc = bufio.NewScanner(os.Stdin)
191-
} else {
192-
gologger.Fatalf("No input provided")
193-
}
194-
195-
for sc.Scan() {
196-
process(sc.Text(), &wg, hp, protocol, scanopts, output)
197-
}
198198

199-
wg.Wait()
199+
wg.Wait()
200200

201-
close(output)
201+
close(output)
202202

203-
wgoutput.Wait()
203+
wgoutput.Wait()
204+
}
204205
}
205206

206207
func process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx.HTTPX, protocol string, scanopts scanOptions, output chan Result) {
@@ -310,6 +311,7 @@ type scanOptions struct {
310311
OutputContentType bool
311312
RequestBody string
312313
Unsafe bool
314+
Pipeline bool
313315
HTTP2Probe bool
314316
}
315317

@@ -459,6 +461,14 @@ retry:
459461
builder.WriteString(" [websocket]")
460462
}
461463

464+
pipeline := false
465+
if scanopts.Pipeline {
466+
pipeline = hp.SupportPipeline(protocol, method, domain, port)
467+
if pipeline {
468+
builder.WriteString(" [pipeline]")
469+
}
470+
}
471+
462472
var http2 bool
463473
// if requested probes for http2
464474
if scanopts.HTTP2Probe {
@@ -496,6 +506,7 @@ retry:
496506
WebSocket: isWebSocket,
497507
TlsData: resp.TlsData,
498508
CspData: resp.CspData,
509+
Pipeline: pipeline,
499510
HTTP2: http2,
500511
Method: method,
501512
}
@@ -518,6 +529,7 @@ type Result struct {
518529
ContentType string `json:"content-type,omitempty"`
519530
TlsData *httpx.TlsData `json:"tls,omitempty"`
520531
CspData *httpx.CspData `json:"csp,omitempty"`
532+
Pipeline bool `json:"pipeline,omitempty"`
521533
HTTP2 bool `json:"http2"`
522534
Method string `json:"method"`
523535
}
@@ -579,6 +591,7 @@ type Options struct {
579591
Unsafe bool
580592
RequestBody string
581593
Debug bool
594+
Pipeline bool
582595
HTTP2Probe bool
583596
}
584597

@@ -625,6 +638,7 @@ func ParseOptions() *Options {
625638
flag.BoolVar(&options.Unsafe, "unsafe", false, "Send raw requests skipping golang normalization")
626639
flag.StringVar(&options.RequestBody, "body", "", "Request Body")
627640
flag.BoolVar(&options.Debug, "debug", false, "Debug mode")
641+
flag.BoolVar(&options.Pipeline, "pipeline", false, "HTTP1.1 Pipeline")
628642
flag.BoolVar(&options.HTTP2Probe, "http2", false, "HTTP2 probe")
629643
flag.Parse()
630644

‎common/httpx/pipeline.go

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package httpx
2+
3+
import (
4+
"crypto/tls"
5+
"fmt"
6+
"net"
7+
"strings"
8+
"time"
9+
)
10+
11+
// TODO: this code must be rewritten with rawhttp
12+
13+
// SupportPipeline checks if the target host supports HTTP1.1 pipelining by sending x probes
14+
// and reading back responses expecting at least 2 with HTTP/1.1 or HTTP/1.0
15+
func (h *HTTPX) SupportPipeline(protocol, method, host string, port int) bool {
16+
addr := host
17+
if port == 0 {
18+
port = 80
19+
if protocol == "https" {
20+
port = 443
21+
}
22+
}
23+
if port > 0 {
24+
addr = fmt.Sprintf("%s:%d", host, port)
25+
}
26+
// dummy method while awaiting for full rawhttp implementation
27+
dummyReq := fmt.Sprintf("%s / HTTP/1.1\nHost: %s\n\n", method, addr)
28+
conn, err := pipelineDial(protocol, addr)
29+
if err != nil {
30+
return false
31+
}
32+
// send some probes
33+
nprobes := 10
34+
for i := 0; i < nprobes; i++ {
35+
if _, err = conn.Write([]byte(dummyReq)); err != nil {
36+
return false
37+
}
38+
}
39+
gotReplies := 0
40+
reply := make([]byte, 1024)
41+
for i := 0; i < nprobes; i++ {
42+
conn.SetReadDeadline(time.Now().Add(1 * time.Second))
43+
if _, err := conn.Read(reply); err != nil {
44+
break
45+
}
46+
47+
// The check is very naive, but it works most of the times
48+
for _, s := range strings.Split(string(reply), "\n\n") {
49+
if strings.Contains(s, "HTTP/1.1") || strings.Contains(s, "HTTP/1.0") {
50+
gotReplies++
51+
}
52+
}
53+
54+
}
55+
56+
// expect at least 2 replies
57+
return gotReplies >= 2
58+
}
59+
60+
func pipelineDial(protocol, addr string) (net.Conn, error) {
61+
// http
62+
if protocol == "http" {
63+
return net.Dial("tcp", addr)
64+
}
65+
66+
// https
67+
return tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true})
68+
}

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ require (
1010
github.com/miekg/dns v1.1.31
1111
github.com/projectdiscovery/gologger v1.0.1
1212
github.com/projectdiscovery/mapcidr v0.0.4
13-
github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84
13+
github.com/projectdiscovery/rawhttp v0.0.0-20200830154207-1d82476d6442
1414
github.com/projectdiscovery/retryablehttp-go v1.0.1
1515
github.com/remeh/sizedwaitgroup v1.0.0
1616
github.com/rs/xid v1.2.1

‎go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs2
2424
github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
2525
github.com/projectdiscovery/mapcidr v0.0.4 h1:2vBSjkmbQASAcO/m2L/dhdulMVu2y9HdyWOrWYJ74rU=
2626
github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0=
27-
github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84 h1:2aO1hZXYAh/UIboBqXyBJ17bHnEWa3y/5rCrIUYqfD0=
28-
github.com/projectdiscovery/rawhttp v0.0.0-20200825153041-19146aae6d84/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
27+
github.com/projectdiscovery/rawhttp v0.0.0-20200830154207-1d82476d6442 h1:06LrG4cDuRRCYnjpRUGSyffOwAMV/ZjunEtygPXnfqg=
28+
github.com/projectdiscovery/rawhttp v0.0.0-20200830154207-1d82476d6442/go.mod h1:RkML6Yq6hf4z2wAUXisa15al4bS+wuJnlhM5ZOfn9k4=
2929
github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNwshuwptuBVYWw9lx8RE=
3030
github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
3131
github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=

0 commit comments

Comments
 (0)