From 87de043597e9d44a3806b5a812476b0bc34f245d Mon Sep 17 00:00:00 2001 From: C-Sto <7466346+C-Sto@users.noreply.github.com> Date: Tue, 16 Oct 2018 23:29:22 +1100 Subject: [PATCH] Fix sending multiple request to same endpoint bug --- librecursebuster/logic.go | 25 ++++++++++++++++--------- librecursebuster/main_test.go | 16 ++++++++-------- librecursebuster/testserver/main.go | 25 +++++++++++++++++++++++-- librecursebuster/ui.go | 5 ++--- main.go | 2 +- 5 files changed, 50 insertions(+), 23 deletions(-) diff --git a/librecursebuster/logic.go b/librecursebuster/logic.go index 3b9048a..99a5438 100644 --- a/librecursebuster/logic.go +++ b/librecursebuster/logic.go @@ -43,7 +43,6 @@ func ManageNewURLs() { //decides on whether to add to the directory list, or add to file output for { candidate := <-gState.Chans.newPagesChan - //check the candidate is an actual URL u, err := url.Parse(strings.TrimSpace(candidate.URL)) @@ -185,8 +184,7 @@ func dirBust(page SpiderPage) { } atomic.StoreUint32(gState.DirbProgress, 0) - - tested := make(map[string]bool) //ensure we don't send things more than once + //ensure we don't send things more than once for _, word := range gState.WordList { //will receive from the channel until it's closed //read words off the channel, and test it OR close out because we wanna skip it if word == "" { @@ -204,31 +202,40 @@ func dirBust(page SpiderPage) { for _, method := range gState.Methods { if len(gState.Extensions) > 0 && gState.Extensions[0] != "" { for _, ext := range gState.Extensions { - if tested[method+page.URL+word+"."+ext] { + gState.CMut.Lock() + if gState.Checked[method+page.URL+word+"."+ext] { + gState.CMut.Unlock() continue } gState.Chans.workersChan <- struct{}{} gState.wg.Add(1) go testURL(method, page.URL+word+"."+ext, gState.Client) - tested[method+page.URL+word+"."+ext] = true + gState.Checked[method+page.URL+word+"."+ext] = true + gState.CMut.Unlock() } } if gState.Cfg.AppendDir { - if tested[method+page.URL+word+"/"] { + gState.CMut.Lock() + if gState.Checked[method+page.URL+word+"/"] { + gState.CMut.Unlock() continue } gState.Chans.workersChan <- struct{}{} gState.wg.Add(1) go testURL(method, page.URL+word+"/", gState.Client) - tested[method+page.URL+word+"/"] = true + gState.Checked[method+page.URL+word+"/"] = true + gState.CMut.Unlock() } - if tested[method+page.URL+word] { + gState.CMut.Lock() + if gState.Checked[method+page.URL+word] { + gState.CMut.Unlock() continue } gState.Chans.workersChan <- struct{}{} gState.wg.Add(1) go testURL(method, page.URL+word, gState.Client) - tested[method+page.URL+word] = true + gState.Checked[method+page.URL+word] = true + gState.CMut.Unlock() //if gState.Cfg.MaxDirs == 1 { atomic.AddUint32(gState.DirbProgress, 1) //} diff --git a/librecursebuster/main_test.go b/librecursebuster/main_test.go index 78e2573..09c9072 100644 --- a/librecursebuster/main_test.go +++ b/librecursebuster/main_test.go @@ -46,7 +46,7 @@ func TestBasicFunctionality(t *testing.T) { finished := make(chan struct{}) cfg := getDefaultConfig() - urlSlice := preSetupTest(cfg, "2001", finished) + urlSlice := preSetupTest(cfg, "2001", finished, t) found := postSetupTest(urlSlice) //waitgroup check (if test times out, the waitgroup is broken... somewhere) @@ -104,7 +104,7 @@ func TestAppendSlash(t *testing.T) { //add an appendslash value to the wordlist that should _only_ be found if the appendslash var is set finished := make(chan struct{}) cfg := getDefaultConfig() - urlSlice := preSetupTest(cfg, "2002", finished) + urlSlice := preSetupTest(cfg, "2002", finished, t) gState.Cfg.AppendDir = true gState.WordList = append(gState.WordList, "appendslash") found := postSetupTest(urlSlice) @@ -121,7 +121,7 @@ func TestBasicAuth(t *testing.T) { finished := make(chan struct{}) cfg := getDefaultConfig() cfg.Auth = "dGVzdDp0ZXN0" - urlSlice := preSetupTest(cfg, "2003", finished) + urlSlice := preSetupTest(cfg, "2003", finished, t) gState.WordList = append(gState.WordList, "basicauth") found := postSetupTest(urlSlice) gState.Wait() @@ -137,7 +137,7 @@ func TestBadCodes(t *testing.T) { finished := make(chan struct{}) cfg := getDefaultConfig() cfg.BadResponses = "404,500" - urlSlice := preSetupTest(cfg, "2004", finished) + urlSlice := preSetupTest(cfg, "2004", finished, t) gState.WordList = append(gState.WordList, "badcode") found := postSetupTest(urlSlice) gState.Wait() @@ -156,7 +156,7 @@ func TestBadHeaders(t *testing.T) { cfg := getDefaultConfig() cfg.BadHeader = ArrayStringFlag{} cfg.BadHeader.Set("X-Bad-Header: test123") - urlSlice := preSetupTest(cfg, "2005", finished) + urlSlice := preSetupTest(cfg, "2005", finished, t) gState.WordList = append(gState.WordList, "badheader") found := postSetupTest(urlSlice) gState.Wait() @@ -175,7 +175,7 @@ func TestAjax(t *testing.T) { cfg := getDefaultConfig() cfg.Ajax = true cfg.Methods = "GET,POST" - urlSlice := preSetupTest(cfg, "2006", finished) + urlSlice := preSetupTest(cfg, "2006", finished, t) gState.WordList = append(gState.WordList, "ajaxonly") gState.WordList = append(gState.WordList, "onlynoajax") gState.WordList = append(gState.WordList, "ajaxpost") @@ -252,7 +252,7 @@ func postSetupTest(urlSlice []string) (found map[string]bool) { return } -func preSetupTest(cfg *Config, servPort string, finished chan struct{}) (urlSlice []string) { +func preSetupTest(cfg *Config, servPort string, finished chan struct{}, t *testing.T) (urlSlice []string) { //Test default functions. Basic dirb should work, and all files should be found as expected //basic state setup @@ -264,7 +264,7 @@ func preSetupTest(cfg *Config, servPort string, finished chan struct{}) (urlSlic //start the test server setup := make(chan struct{}) - go testserver.Start(servPort, finished, setup) + go testserver.Start(servPort, finished, setup, t) <-setup //whoa, concurrency sucks??? //test URL globalState.Cfg.URL = localURL + servPort diff --git a/librecursebuster/testserver/main.go b/librecursebuster/testserver/main.go index 872657d..d457e2d 100644 --- a/librecursebuster/testserver/main.go +++ b/librecursebuster/testserver/main.go @@ -5,6 +5,8 @@ import ( "net/http" "strings" "sync" + "testing" + "time" ) /*wordlist @@ -56,7 +58,8 @@ func handler(w http.ResponseWriter, r *http.Request) { //if it's been visited more than once, instant fail if visited[r.Method+":"+r.URL.Path] && r.URL.Path != "/" { //we can visit the base url more than once - //panic("Path visited more than once: " + r.Method + ":" + r.URL.Path) + tes.Fail() + panic("Path visited more than once: " + r.Method + ":" + r.URL.Path) } visited[r.Method+":"+r.URL.Path] = true vMut.Unlock() @@ -172,13 +175,31 @@ func handler(w http.ResponseWriter, r *http.Request) { fmt.Println(r.Method, r.URL, respCode, bod) } +var tes *testing.T + //Start starts the test HTTP server -func Start(port string, finishedTest, setup chan struct{}) { +func Start(port string, finishedTest, setup chan struct{}, t *testing.T) { s := http.NewServeMux() visited = make(map[string]bool) vMut = &sync.RWMutex{} + tes = t s.HandleFunc("/", handler) go http.ListenAndServe("127.0.0.1:"+port, s) + + for { + time.Sleep(time.Second * 1) + resp, err := http.Get("http://localhost:" + port) + if err != nil { + continue + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + continue + } + // Reached this point: server is up and running! + break + } + close(setup) <-finishedTest //this is an ultra gross hack :( } diff --git a/librecursebuster/ui.go b/librecursebuster/ui.go index 7d275ec..efebac9 100644 --- a/librecursebuster/ui.go +++ b/librecursebuster/ui.go @@ -11,7 +11,6 @@ import ( //StartUI is called to begin the UI... stuff func (s *State) StartUI(uiWG *sync.WaitGroup, quitChan chan struct{}) { g, err := ui.NewGui(ui.OutputNormal) - g.Mouse = true if err != nil { panic(err) } @@ -47,7 +46,7 @@ func (s *State) StartUI(uiWG *sync.WaitGroup, quitChan chan struct{}) { err = s.ui.SetKeybinding("", ui.KeyArrowDown, ui.ModNone, scrollDown) if err != nil { panic(err) - } + } /* Mouse stuff broke copying out of the terminal... not ideal err = s.ui.SetKeybinding("", ui.MouseWheelUp, ui.ModNone, scrollUp) if err != nil { panic(err) @@ -55,7 +54,7 @@ func (s *State) StartUI(uiWG *sync.WaitGroup, quitChan chan struct{}) { err = s.ui.SetKeybinding("", ui.MouseWheelDown, ui.ModNone, scrollDown) if err != nil { panic(err) - } + }*/ uiWG.Done() err = s.ui.MainLoop() if err != nil && err != ui.ErrQuit { diff --git a/main.go b/main.go index 074ce88..2cfea53 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( "github.com/fatih/color" ) -const version = "1.5.10" +const version = "1.5.11" func main() { if runtime.GOOS == "windows" { //lol goos