Skip to content

Commit baca6c6

Browse files
lanratlanrat
committed
saves zones in parallel
1 parent dc31b0f commit baca6c6

File tree

7 files changed

+311
-117
lines changed

7 files changed

+311
-117
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
allxfr

‎axfr.go

+86
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package main
2+
3+
import (
4+
"bufio"
5+
"compress/gzip"
6+
"fmt"
7+
"log"
8+
"net"
9+
"os"
10+
11+
"github.com/miekg/dns"
12+
)
13+
14+
func axfr(zone, nameserver string, ip net.IP) error {
15+
if *verbose {
16+
log.Printf("Trying AXFR: %s %s %s", zone, nameserver, ip.String())
17+
}
18+
m := new(dns.Msg)
19+
m.SetQuestion(zone, dns.TypeAXFR)
20+
21+
ipString := ip.String()
22+
if ip.To4() == nil {
23+
ipString = fmt.Sprintf("[%s]", ipString)
24+
}
25+
26+
t := new(dns.Transfer)
27+
env, err := t.In(m, fmt.Sprintf("%s:53", ipString))
28+
if err != nil {
29+
// skip on this error
30+
err = fmt.Errorf("transfer error from %s (%s): %w", nameserver, ip.String(), err)
31+
if *verbose {
32+
log.Print(err)
33+
}
34+
return nil
35+
}
36+
37+
// get ready to save file
38+
filename := fmt.Sprintf("%s/%s_%s_%s.zone.gz", *saveDir, zone, nameserver, ip.String())
39+
filenameTmp := fmt.Sprintf("%s.tmp", filename)
40+
fi, err := os.Create(filenameTmp)
41+
if err != nil {
42+
return err
43+
}
44+
gf := gzip.NewWriter(fi)
45+
fw := bufio.NewWriter(gf)
46+
47+
var envelope, record int
48+
for e := range env {
49+
if e.Error != nil {
50+
// skip on this error
51+
err = fmt.Errorf("transfer envelope error from %v: %w", nameserver, e.Error)
52+
if *verbose {
53+
log.Print(err)
54+
}
55+
break
56+
}
57+
for _, r := range e.RR {
58+
//fmt.Printf("%s\n", r)
59+
_, err = fw.WriteString(fmt.Sprintf("%s\n", r.String()))
60+
if err != nil {
61+
fw.Flush()
62+
gf.Close()
63+
fi.Close()
64+
os.Remove(filenameTmp)
65+
return err
66+
}
67+
68+
}
69+
record += len(e.RR)
70+
envelope++
71+
}
72+
fw.Flush()
73+
gf.Close()
74+
fi.Close()
75+
if record > 1 {
76+
log.Printf("%s %s (%s) xfr size: %d records (envelopes %d)\n", zone, nameserver, ip.String(), record, envelope)
77+
err = os.Rename(filenameTmp, filename)
78+
} else {
79+
err = os.Remove(filenameTmp)
80+
}
81+
if err != nil {
82+
return err
83+
}
84+
85+
return nil
86+
}

‎go.mod

+5-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
1-
module allaxfr
1+
module allxfr
22

33
go 1.13
44

5-
require github.com/miekg/dns v1.1.22
5+
require (
6+
github.com/miekg/dns v1.1.22
7+
golang.org/x/sync v0.0.0-20190423024810-112230192c58
8+
)

‎go.sum

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
77
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
88
golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
99
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
10+
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
1011
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
1112
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
1213
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

‎main.go

+64-115
Original file line numberDiff line numberDiff line change
@@ -1,149 +1,98 @@
11
package main
22

33
import (
4+
"flag"
45
"fmt"
56
"log"
6-
"net"
7+
"os"
78

89
"github.com/miekg/dns"
10+
"golang.org/x/sync/errgroup"
911
)
1012

11-
type root struct {
12-
// map of TLDs to nameservers
13-
tld map[string][]string
14-
// map of nameservers to ipv4 and ipv6
15-
ip map[string][]net.IP
16-
}
13+
var (
14+
initialNameserver = flag.String("ns", "", "initial nameserver to use to get the root")
15+
parallel = flag.Uint("parallel", 10, "number of parallel zone transfers to perform")
16+
saveDir = flag.String("out", ".", "directory to save found zones in")
17+
verbose = flag.Bool("verbose", false, "enable verbose output")
18+
)
1719

18-
func (r *root) AddTLD(tld, nameserver string) {
19-
if r.tld == nil {
20-
r.tld = make(map[string][]string)
21-
}
22-
_, ok := r.tld[tld]
23-
if !ok {
24-
r.tld[tld] = make([]string, 0, 4)
20+
func main() {
21+
flag.Parse()
22+
localNameserver, err := getInitialNameserver()
23+
check(err)
24+
if *verbose {
25+
log.Printf("Using initial nameserver %s", localNameserver)
2526
}
26-
r.tld[tld] = append(r.tld[tld], nameserver)
27-
}
27+
rootNameservers, err := getRootServers(localNameserver)
28+
check(err)
2829

29-
func (r *root) AddIP(nameserver string, ip net.IP) {
30-
if r.ip == nil {
31-
r.ip = make(map[string][]net.IP)
30+
var root zone
31+
// not all the root nameservers allow AXFR, try them until we find one that does
32+
for _, ns := range rootNameservers {
33+
if *verbose {
34+
log.Printf("Trying root nameserver %s", ns)
35+
}
36+
root, err = rootAXFR(ns)
37+
if err == nil {
38+
break
39+
}
3240
}
33-
_, ok := r.ip[nameserver]
34-
if !ok {
35-
r.ip[nameserver] = make([]net.IP, 0, 4)
41+
if root.CountNS() == 0 {
42+
log.Fatal("Got empty root zone")
3643
}
37-
r.ip[nameserver] = append(r.ip[nameserver], ip)
38-
}
3944

40-
func (r *root) Print() {
41-
fmt.Println("NS:")
42-
for zone := range r.tld {
43-
fmt.Printf("%s\n", zone)
44-
for i := range r.tld[zone] {
45-
fmt.Printf("\t%s\n", r.tld[zone][i])
46-
}
47-
}
48-
fmt.Println("IP:")
49-
for ns := range r.ip {
50-
fmt.Printf("%s\n", ns)
51-
for i := range r.ip[ns] {
52-
fmt.Printf("\t%s\n", r.ip[ns][i])
53-
}
45+
// create outpout dir if does not exist
46+
if _, err := os.Stat(*saveDir); os.IsNotExist(err) {
47+
err = os.MkdirAll(*saveDir, os.ModePerm)
48+
check(err)
5449
}
55-
}
5650

57-
func (r *root) PrintTree() {
58-
fmt.Println("Zones:")
59-
for zone := range r.tld {
60-
fmt.Printf("%s\n", zone)
61-
for i := range r.tld[zone] {
62-
fmt.Printf("\t%s\n", r.tld[zone][i])
63-
for j := range r.ip[r.tld[zone][i]] {
64-
fmt.Printf("\t\t%s\n", r.ip[r.tld[zone][i]][j])
65-
}
66-
}
51+
if *verbose {
52+
root.PrintTree()
6753
}
68-
}
54+
rootChan := root.GetNsIPChan()
55+
var g errgroup.Group
6956

70-
func main() {
71-
server, err := getRootServer()
72-
check(err)
57+
// start workers
58+
for i := uint(0); i < *parallel; i++ {
59+
g.Go(func() error { return worker(rootChan) })
60+
}
7361

74-
log.Printf("using server %s", server)
75-
err = rootAXFR(server)
62+
err = g.Wait()
7663
check(err)
77-
log.Printf("done")
7864
}
7965

80-
func check(err error) {
81-
if err != nil {
82-
log.Fatal(err)
83-
}
84-
}
85-
86-
func rootAXFR(ns string) error {
87-
m := new(dns.Msg)
88-
m.SetQuestion(".", dns.TypeAXFR)
89-
90-
t := new(dns.Transfer)
91-
92-
var root root
93-
94-
env, err := t.In(m, fmt.Sprintf("%s:53", ns))
95-
if err != nil {
96-
return err
97-
}
98-
var envelope, record int
99-
for e := range env {
100-
//fmt.Println("envelope loop") // 108, contains abut 200 records
101-
if e.Error != nil {
102-
return e.Error
66+
func worker(c chan nsip) error {
67+
for {
68+
r, more := <-c
69+
if !more {
70+
return nil
10371
}
104-
for _, r := range e.RR {
105-
//fmt.Println("RR loop") // 22077
106-
//fmt.Printf("IAN: %s\n", r)
107-
switch t := r.(type) {
108-
case *dns.A:
109-
root.AddIP(t.Hdr.Name, t.A)
110-
case *dns.AAAA:
111-
root.AddIP(t.Hdr.Name, t.AAAA)
112-
case *dns.NS:
113-
root.AddTLD(t.Hdr.Name, t.Ns)
114-
}
72+
err := axfr(r.domain, r.ns, r.ip)
73+
if err != nil {
74+
return err
11575
}
116-
record += len(e.RR)
117-
envelope++
11876
}
119-
log.Printf("\n;; xfr size: %d records (envelopes %d)\n", record, envelope)
120-
121-
root.PrintTree()
122-
return nil
12377
}
12478

125-
var ErrNoRoot = fmt.Errorf("Unable to find Root Server")
126-
127-
func getRootServer() (string, error) {
128-
// get root server from local DNS
129-
conf, err := dns.ClientConfigFromFile("/etc/resolv.conf")
79+
func check(err error) {
13080
if err != nil {
131-
return "", err
81+
log.Fatal(err)
13282
}
133-
localserver := fmt.Sprintf("%s:%s", conf.Servers[0], conf.Port)
83+
}
13484

135-
// get root servers
136-
m := new(dns.Msg)
137-
m.SetQuestion(".", dns.TypeNS)
138-
in, err := dns.Exchange(m, localserver)
139-
if err != nil {
140-
return "", err
141-
}
142-
//fmt.Println(in)
143-
for _, a := range in.Answer {
144-
if ns, ok := a.(*dns.NS); ok {
145-
return ns.Ns, nil
85+
func getInitialNameserver() (string, error) {
86+
var server string
87+
if len(*initialNameserver) == 0 {
88+
// get root server from local DNS
89+
conf, err := dns.ClientConfigFromFile("/etc/resolv.conf")
90+
if err != nil {
91+
return "", err
14692
}
93+
server = fmt.Sprintf("%s:%s", conf.Servers[0], conf.Port)
94+
} else {
95+
server = fmt.Sprintf("%s:53", *initialNameserver)
14796
}
148-
return "", ErrNoRoot
97+
return server, nil
14998
}

‎root.go

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"log"
6+
7+
"github.com/miekg/dns"
8+
)
9+
10+
func getRootServers(localserver string) ([]string, error) {
11+
out := make([]string, 0, 4)
12+
// get root servers
13+
m := new(dns.Msg)
14+
m.SetQuestion(".", dns.TypeNS)
15+
in, err := dns.Exchange(m, localserver)
16+
if err != nil {
17+
return out, err
18+
}
19+
for _, a := range in.Answer {
20+
if ns, ok := a.(*dns.NS); ok {
21+
out = append(out, ns.Ns)
22+
}
23+
}
24+
if len(out) == 0 {
25+
return out, fmt.Errorf("Unable to find Root Server")
26+
27+
}
28+
return out, nil
29+
}
30+
31+
func rootAXFR(ns string) (zone, error) {
32+
m := new(dns.Msg)
33+
m.SetQuestion(".", dns.TypeAXFR)
34+
35+
t := new(dns.Transfer)
36+
37+
var root zone
38+
39+
env, err := t.In(m, fmt.Sprintf("%s:53", ns))
40+
if err != nil {
41+
return root, fmt.Errorf("transfer error from %v: %w", ns, err)
42+
}
43+
var envelope, record int
44+
for e := range env {
45+
if e.Error != nil {
46+
return root, fmt.Errorf("transfer envelope error from %v: %w", ns, e.Error)
47+
}
48+
for _, r := range e.RR {
49+
switch t := r.(type) {
50+
case *dns.A:
51+
root.AddIP(t.Hdr.Name, t.A)
52+
case *dns.AAAA:
53+
root.AddIP(t.Hdr.Name, t.AAAA)
54+
case *dns.NS:
55+
root.AddTLD(t.Hdr.Name, t.Ns)
56+
}
57+
}
58+
record += len(e.RR)
59+
envelope++
60+
}
61+
log.Printf("ROOT xfr size: %d records (envelopes %d)\n", record, envelope)
62+
63+
return root, nil
64+
}

0 commit comments

Comments
 (0)