Shoco lib, better compression, license

Author: coyove

.gitignore

@@ -0,0 +1 @@
+top-1m.csv

LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 coyove <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

config/config.go

@@ -16,11 +16,12 @@ var (
 	G_Password = flag.String("p", "password", "proxy password")
 	G_Upstream = flag.String("up", "", "upstream server address (e.g. 127.0.0.1:8100)")
 	G_Local    = flag.String("l", ":8100", "local listening")
-	G_Dummies  = flag.String("dummy", "", "dummy hosts, separated by |")
+	G_Dummies  = flag.String("dummy", "china", "dummy hosts, separated by |")
 
 	G_Debug      = flag.Bool("debug", false, "debug mode")
 	G_UnsafeHttp = flag.Bool("disable-sh", false, "do not encrypt http content")
 	G_NoPA       = flag.Bool("disable-pa", false, "disable proxy authentication")
+	G_NoShoco    = flag.Bool("disable-shoco", false, "disable shoco compression")
 	G_ProxyAll   = flag.Bool("all", false, "proxy Chinese websites")
 
 	G_SuppressSocketReadWriteError = flag.Bool("ssrwe", false, "suppress socket read/write error")
@@ -80,6 +81,7 @@ func LoadConfig(path string) {
 	setBool(G_UnsafeHttp, m["unsafehttp"])
 	setBool(G_SuppressSocketReadWriteError, m["ssrwe"])
 	setBool(G_NoPA, m["disablepa"])
+	setBool(G_NoShoco, m["disableshoco"])
 	setBool(G_ProxyAll, m["proxyall"])
 
 	setInt(G_DNSCacheEntries, m["dnscache"])

lookup/lookup.go

@@ -82,7 +82,7 @@ func IsPrivateIP(ip string) bool {
 func LookupIP(host string) string {
 	ip, err := net.ResolveIPAddr("ip4", host)
 	if err != nil {
-		logg.L("dns lookup: ", err)
+		logg.L("[DNS] ", err)
 		return ""
 	}
 

lru/lru.go

@@ -3,6 +3,8 @@ package lru
 import (
 	"container/list"
 	"sync"
+
+	"../logg"
 )
 
 type Cache struct {
@@ -50,9 +52,11 @@ func (c *Cache) Clear() {
 func (c *Cache) Info(callback func(Key, interface{}, int64)) {
 	c.RLock()
 	f := c.ll.Front()
+	r := 0
 
-	for {
-		if f == nil {
+	for f != nil {
+		if r++; r > c.MaxEntries {
+			logg.E("something bad happened: we had a loop")
 			break
 		}
 
@@ -84,7 +88,7 @@ func (c *Cache) Add(key Key, value interface{}) {
 		return
 	}
 
-	ele := c.ll.PushFront(&entry{key, value, 0})
+	ele := c.ll.PushFront(&entry{key, value, 1})
 	c.cache[key] = ele
 	if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
 		c.removeOldest()
@@ -125,8 +129,8 @@ func (c *Cache) Remove(key Key) {
 
 // Len returns the number of items in the cache.
 func (c *Cache) Len() int {
-	c.Lock()
-	defer c.Unlock()
+	c.RLock()
+	defer c.RUnlock()
 
 	if c.cache == nil {
 		return 0

proxy/proxy.go

@@ -88,9 +88,9 @@ func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request)
 				return
 			}
 
-			rkey := RandomKey()
+			rkey, rhost := RandomKey(), dummyHosts[NewRand().Intn(len(dummyHosts))]
 			upstreamConn.Write([]byte(fmt.Sprintf(
-				"CONNECT www.baidu.com HTTP/1.1\r\nHost: www.baidu.com\r\nX-Forwarded-Host: %s\r\n%s: %s\r\n\r\n", host, rkeyHeader, rkey)))
+				"CONNECT www.%s.com HTTP/1.1\r\nHost: www.%s.com\r\nX-Forwarded-Host: %s\r\n%s: %s\r\n\r\n", rhost, rhost, host, rkeyHeader, rkey)))
 
 			TwoWayBridge(proxyClient, upstreamConn, rkey)
 		} else {
@@ -219,15 +219,15 @@ func (proxy *ProxyHttpServer) CanDirectConnect(host string) bool {
 	resp, err := client.Do(req)
 
 	if err != nil {
-		logg.W("host lookup: ", err)
+		logg.W("[REMOTE LOOKUP] ", err)
 		return maybeChinese
 	}
 
 	ipbuf, err := ioutil.ReadAll(resp.Body)
 	tryClose(resp.Body)
 
 	if err != nil {
-		logg.W("host lookup: ", err)
+		logg.W("[REMOTE LOOKUP] ", err)
 		return maybeChinese
 	}
 

proxy/util.go

@@ -4,6 +4,7 @@ import (
 	. "../config"
 	"../logg"
 	"../lru"
+	"../shoco"
 
 	"bytes"
 	"crypto/tls"
@@ -17,6 +18,7 @@ import (
 	"net/http"
 	"net/url"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -31,7 +33,7 @@ var (
 	dnsHeaderID = "X-Host-Lookup-ID"
 	dnsHeader   = "X-Host-Lookup"
 
-	hostHeadHolder  = "%s.%s.com:433"
+	hostHeadHolder  = "%s.%s.com:%d"
 	dummyHosts      = append([]string{"baidu", "qq", "taobao", "sina", "163", "youku"}, strings.Split(*G_Dummies, "|")...)
 	hostHeadExtract = regexp.MustCompile(`(\S+)\.(?:` + strings.Join(dummyHosts, "|") + `)\.com`)
 	urlExtract      = regexp.MustCompile(`\?q=(\S+)$`)
@@ -149,13 +151,47 @@ func DecryptRequest(req *http.Request) string {
 	return rkey
 }
 
+func SplitHostPort(host string) (string, int) {
+	if idx := strings.Index(host, ":"); idx > 0 {
+		n, err := strconv.Atoi(host[idx+1:])
+		if err != nil {
+			logg.E("cannot split: ", host)
+			return host, 80
+		}
+
+		return host[:idx], n
+	} else {
+		return host, 80
+	}
+}
+
 func EncryptHost(host string) string {
-	return fmt.Sprintf(hostHeadHolder, Skip32EncodeString(G_KeyBytes, host), dummyHosts[NewRand().Intn(len(dummyHosts))])
+	h, p := SplitHostPort(host)
+	dummy := dummyHosts[NewRand().Intn(len(dummyHosts))]
+
+	if *G_NoShoco {
+		return fmt.Sprintf(hostHeadHolder, Skip32EncodeString(G_KeyBytes, h), dummy, p)
+	}
+
+	x := Base36Encode(Skip32Encode(G_KeyBytes, shoco.CompressHost(h), false))
+	return fmt.Sprintf(hostHeadHolder, x, dummy, p)
 }
 
 func DecryptHost(host string) string {
-	if p := hostHeadExtract.FindStringSubmatch(host); len(p) > 1 {
-		return Skip32DecodeString(G_KeyBytes, p[1])
+	defer func() {
+		if r := recover(); r != nil {
+			logg.E("[SHOCO] - ", r)
+		}
+	}()
+
+	h, p := SplitHostPort(host)
+
+	if s := hostHeadExtract.FindStringSubmatch(h); len(s) > 1 {
+		if *G_NoShoco {
+			return Skip32DecodeString(G_KeyBytes, s[1]) + ":" + strconv.Itoa(p)
+		}
+
+		return shoco.DecompressHost(Skip32Decode(G_KeyBytes, Base36Decode(s[1]), false)) + ":" + strconv.Itoa(p)
 	}
 
 	return ""

shoco/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Christian Schramm <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+