Feat: remove geo files from repo & refine tests (#869)

This commit is contained in:
Loyalsoldier 2021-04-07 03:08:03 +08:00 committed by GitHub
parent 3f9b5e243e
commit 10ca68f277
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 195 additions and 11388 deletions

View File

@ -118,6 +118,11 @@ jobs:
mv v2ray v2ray.exe
mv v2ctl v2ctl.exe
- name: Download geo files
run: |
wget -O release/config/geoip.dat "https://raw.githubusercontent.com/v2fly/geoip/release/geoip.dat"
wget -O release/config/geosite.dat "https://raw.githubusercontent.com/v2fly/domain-list-community/release/dlc.dat"
- name: Prepare package
run: cp -v ./release/config/*.* ./build_assets

View File

@ -1,25 +0,0 @@
name: Update Geofiles
on:
schedule:
- cron: "0 0 * * FRI"
jobs:
update:
if: github.repository == 'v2fly/v2ray-core'
runs-on: ubuntu-latest
steps:
- name: Checkout codebase
uses: actions/checkout@v2
- name: Download
run: |
curl -L -o release/config/geoip.dat "https://github.com/v2fly/geoip/raw/release/geoip.dat"
curl -L -o release/config/geosite.dat "https://github.com/v2fly/domain-list-community/raw/release/dlc.dat"
- name: push
run: |
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
git commit -am "update geoip, geosite"
git push -v --progress

2
.gitignore vendored
View File

@ -25,4 +25,4 @@ vprotogen
!infra/vprotogen/
errorgen
!common/errors/errorgen/
infra/conf/*.dat
*.dat

View File

@ -1,11 +1,13 @@
package router_test
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/proto"
"github.com/v2fly/v2ray-core/v4/app/router"
"github.com/v2fly/v2ray-core/v4/common"
@ -18,11 +20,18 @@ func init() {
wd, err := os.Getwd()
common.Must(err)
if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && os.IsNotExist(err) {
common.Must(filesystem.CopyFile(platform.GetAssetLocation("geoip.dat"), filepath.Join(wd, "..", "..", "release", "config", "geoip.dat")))
}
if _, err := os.Stat(platform.GetAssetLocation("geosite.dat")); err != nil && os.IsNotExist(err) {
common.Must(filesystem.CopyFile(platform.GetAssetLocation("geosite.dat"), filepath.Join(wd, "..", "..", "release", "config", "geosite.dat")))
tempPath := filepath.Join(wd, "..", "..", "testing", "temp")
geoipPath := filepath.Join(tempPath, "geoip.dat")
os.Setenv("v2ray.location.asset", tempPath)
if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && errors.Is(err, os.ErrNotExist) {
if _, err := os.Stat(geoipPath); err != nil && errors.Is(err, os.ErrNotExist) {
common.Must(os.MkdirAll(tempPath, 0755))
geoipBytes, err := common.FetchHTTPContent(geoipURL)
common.Must(err)
common.Must(filesystem.WriteFile(geoipPath, geoipBytes))
}
}
}
@ -195,7 +204,7 @@ func loadGeoIP(country string) ([]*router.CIDR, error) {
}
for _, geoip := range geoipList.Entry {
if geoip.CountryCode == country {
if strings.EqualFold(geoip.CountryCode, country) {
return geoip.Cidr, nil
}
}

View File

@ -1,16 +1,17 @@
package router_test
import (
"errors"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/proto"
. "github.com/v2fly/v2ray-core/v4/app/router"
"github.com/v2fly/v2ray-core/v4/app/router"
"github.com/v2fly/v2ray-core/v4/common"
"github.com/v2fly/v2ray-core/v4/common/errors"
"github.com/v2fly/v2ray-core/v4/common/net"
"github.com/v2fly/v2ray-core/v4/common/platform"
"github.com/v2fly/v2ray-core/v4/common/platform/filesystem"
@ -25,11 +26,28 @@ func init() {
wd, err := os.Getwd()
common.Must(err)
if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && os.IsNotExist(err) {
common.Must(filesystem.CopyFile(platform.GetAssetLocation("geoip.dat"), filepath.Join(wd, "..", "..", "release", "config", "geoip.dat")))
tempPath := filepath.Join(wd, "..", "..", "testing", "temp")
geoipPath := filepath.Join(tempPath, "geoip.dat")
geositePath := filepath.Join(tempPath, "geosite.dat")
os.Setenv("v2ray.location.asset", tempPath)
if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && errors.Is(err, os.ErrNotExist) {
if _, err := os.Stat(geoipPath); err != nil && errors.Is(err, os.ErrNotExist) {
common.Must(os.MkdirAll(tempPath, 0755))
geoipBytes, err := common.FetchHTTPContent(geoipURL)
common.Must(err)
common.Must(filesystem.WriteFile(geoipPath, geoipBytes))
}
}
if _, err := os.Stat(platform.GetAssetLocation("geosite.dat")); err != nil && os.IsNotExist(err) {
common.Must(filesystem.CopyFile(platform.GetAssetLocation("geosite.dat"), filepath.Join(wd, "..", "..", "release", "config", "geosite.dat")))
if _, err := os.Stat(platform.GetAssetLocation("geosite.dat")); err != nil && errors.Is(err, os.ErrNotExist) {
if _, err := os.Stat(geositePath); err != nil && errors.Is(err, os.ErrNotExist) {
common.Must(os.MkdirAll(tempPath, 0755))
geositeBytes, err := common.FetchHTTPContent(geositeURL)
common.Must(err)
common.Must(filesystem.WriteFile(geositePath, geositeBytes))
}
}
}
@ -56,23 +74,23 @@ func TestRoutingRule(t *testing.T) {
}
cases := []struct {
rule *RoutingRule
rule *router.RoutingRule
test []ruleTest
}{
{
rule: &RoutingRule{
Domain: []*Domain{
rule: &router.RoutingRule{
Domain: []*router.Domain{
{
Value: "v2fly.org",
Type: Domain_Plain,
Type: router.Domain_Plain,
},
{
Value: "google.com",
Type: Domain_Domain,
Type: router.Domain_Domain,
},
{
Value: "^facebook\\.com$",
Type: Domain_Regex,
Type: router.Domain_Regex,
},
},
},
@ -108,8 +126,8 @@ func TestRoutingRule(t *testing.T) {
},
},
{
rule: &RoutingRule{
Cidr: []*CIDR{
rule: &router.RoutingRule{
Cidr: []*router.CIDR{
{
Ip: []byte{8, 8, 8, 8},
Prefix: 32,
@ -144,10 +162,10 @@ func TestRoutingRule(t *testing.T) {
},
},
{
rule: &RoutingRule{
Geoip: []*GeoIP{
rule: &router.RoutingRule{
Geoip: []*router.GeoIP{
{
Cidr: []*CIDR{
Cidr: []*router.CIDR{
{
Ip: []byte{8, 8, 8, 8},
Prefix: 32,
@ -184,8 +202,8 @@ func TestRoutingRule(t *testing.T) {
},
},
{
rule: &RoutingRule{
SourceCidr: []*CIDR{
rule: &router.RoutingRule{
SourceCidr: []*router.CIDR{
{
Ip: []byte{192, 168, 0, 0},
Prefix: 16,
@ -204,7 +222,7 @@ func TestRoutingRule(t *testing.T) {
},
},
{
rule: &RoutingRule{
rule: &router.RoutingRule{
UserEmail: []string{
"admin@v2fly.org",
},
@ -225,7 +243,7 @@ func TestRoutingRule(t *testing.T) {
},
},
{
rule: &RoutingRule{
rule: &router.RoutingRule{
Protocol: []string{"http"},
},
test: []ruleTest{
@ -236,7 +254,7 @@ func TestRoutingRule(t *testing.T) {
},
},
{
rule: &RoutingRule{
rule: &router.RoutingRule{
InboundTag: []string{"test", "test1"},
},
test: []ruleTest{
@ -251,7 +269,7 @@ func TestRoutingRule(t *testing.T) {
},
},
{
rule: &RoutingRule{
rule: &router.RoutingRule{
PortList: &net.PortList{
Range: []*net.PortRange{
{From: 443, To: 443},
@ -279,7 +297,7 @@ func TestRoutingRule(t *testing.T) {
},
},
{
rule: &RoutingRule{
rule: &router.RoutingRule{
SourcePortList: &net.PortList{
Range: []*net.PortRange{
{From: 123, To: 123},
@ -307,7 +325,7 @@ func TestRoutingRule(t *testing.T) {
},
},
{
rule: &RoutingRule{
rule: &router.RoutingRule{
Protocol: []string{"http"},
Attributes: "attrs[':path'].startswith('/test')",
},
@ -333,32 +351,31 @@ func TestRoutingRule(t *testing.T) {
}
}
func loadGeoSite(country string) ([]*Domain, error) {
func loadGeoSite(country string) ([]*router.Domain, error) {
geositeBytes, err := filesystem.ReadAsset("geosite.dat")
if err != nil {
return nil, err
}
var geositeList GeoSiteList
var geositeList router.GeoSiteList
if err := proto.Unmarshal(geositeBytes, &geositeList); err != nil {
return nil, err
}
for _, site := range geositeList.Entry {
if site.CountryCode == country {
if strings.EqualFold(site.CountryCode, country) {
return site.Domain, nil
}
}
return nil, errors.New("country not found: " + country)
}
func TestChinaSites(t *testing.T) {
domains, err := loadGeoSite("CN")
common.Must(err)
matcher, err := NewDomainMatcher(domains)
matcher, err := router.NewDomainMatcher(domains)
common.Must(err)
acMatcher, err := NewMphMatcherGroup(domains)
acMatcher, err := router.NewMphMatcherGroup(domains)
common.Must(err)
type TestCase struct {
@ -403,7 +420,7 @@ func BenchmarkMphDomainMatcher(b *testing.B) {
domains, err := loadGeoSite("CN")
common.Must(err)
matcher, err := NewMphMatcherGroup(domains)
matcher, err := router.NewMphMatcherGroup(domains)
common.Must(err)
type TestCase struct {
@ -445,7 +462,7 @@ func BenchmarkDomainMatcher(b *testing.B) {
domains, err := loadGeoSite("CN")
common.Must(err)
matcher, err := NewDomainMatcher(domains)
matcher, err := router.NewDomainMatcher(domains)
common.Must(err)
type TestCase struct {
@ -484,12 +501,12 @@ func BenchmarkDomainMatcher(b *testing.B) {
}
func BenchmarkMultiGeoIPMatcher(b *testing.B) {
var geoips []*GeoIP
var geoips []*router.GeoIP
{
ips, err := loadGeoIP("CN")
common.Must(err)
geoips = append(geoips, &GeoIP{
geoips = append(geoips, &router.GeoIP{
CountryCode: "CN",
Cidr: ips,
})
@ -498,7 +515,7 @@ func BenchmarkMultiGeoIPMatcher(b *testing.B) {
{
ips, err := loadGeoIP("JP")
common.Must(err)
geoips = append(geoips, &GeoIP{
geoips = append(geoips, &router.GeoIP{
CountryCode: "JP",
Cidr: ips,
})
@ -507,7 +524,7 @@ func BenchmarkMultiGeoIPMatcher(b *testing.B) {
{
ips, err := loadGeoIP("CA")
common.Must(err)
geoips = append(geoips, &GeoIP{
geoips = append(geoips, &router.GeoIP{
CountryCode: "CA",
Cidr: ips,
})
@ -516,13 +533,13 @@ func BenchmarkMultiGeoIPMatcher(b *testing.B) {
{
ips, err := loadGeoIP("US")
common.Must(err)
geoips = append(geoips, &GeoIP{
geoips = append(geoips, &router.GeoIP{
CountryCode: "US",
Cidr: ips,
})
}
matcher, err := NewMultiGeoIPMatcher(geoips, false)
matcher, err := router.NewMultiGeoIPMatcher(geoips, false)
common.Must(err)
ctx := withOutbound(&session.Outbound{Target: net.TCPDestination(net.ParseAddress("8.8.8.8"), 80)})

View File

@ -0,0 +1,6 @@
package router_test
const (
geoipURL = "https://raw.githubusercontent.com/v2fly/geoip/release/geoip.dat"
geositeURL = "https://raw.githubusercontent.com/v2fly/domain-list-community/release/dlc.dat"
)

View File

@ -6,9 +6,12 @@ import (
"fmt"
"go/build"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
"github.com/v2fly/v2ray-core/v4/common/errors"
)
@ -123,3 +126,39 @@ func GetGOPATH() string {
}
return GOPATH
}
// FetchHTTPContent dials http(s) for remote content
func FetchHTTPContent(target string) ([]byte, error) {
parsedTarget, err := url.Parse(target)
if err != nil {
return nil, newError("invalid URL: ", target).Base(err)
}
if s := strings.ToLower(parsedTarget.Scheme); s != "http" && s != "https" {
return nil, newError("invalid scheme: ", parsedTarget.Scheme)
}
client := &http.Client{
Timeout: 30 * time.Second,
}
resp, err := client.Do(&http.Request{
Method: "GET",
URL: parsedTarget,
Close: true,
})
if err != nil {
return nil, newError("failed to dial to ", target).Base(err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, newError("unexpected HTTP status code: ", resp.StatusCode)
}
content, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, newError("failed to read HTTP response").Base(err)
}
return content, nil
}

View File

@ -10,10 +10,16 @@ import (
type FileReaderFunc func(path string) (io.ReadCloser, error)
type FileWriterFunc func(path string) (io.WriteCloser, error)
var NewFileReader FileReaderFunc = func(path string) (io.ReadCloser, error) {
return os.Open(path)
}
var NewFileWriter FileWriterFunc = func(path string) (io.WriteCloser, error) {
return os.Create(path)
}
func ReadFile(path string) ([]byte, error) {
reader, err := NewFileReader(path)
if err != nil {
@ -24,6 +30,16 @@ func ReadFile(path string) ([]byte, error) {
return buf.ReadAllToBytes(reader)
}
func WriteFile(path string, payload []byte) error {
writer, err := NewFileWriter(path)
if err != nil {
return err
}
defer writer.Close()
return buf.WriteAllBytes(writer, payload)
}
func ReadAsset(file string) ([]byte, error) {
return ReadFile(platform.GetAssetLocation(file))
}

View File

@ -3,6 +3,7 @@
package platform
import (
"errors"
"os"
"path/filepath"
)
@ -32,7 +33,7 @@ func GetAssetLocation(file string) string {
filepath.Join("/usr/share/v2ray/", file),
filepath.Join("/opt/share/v2ray/", file),
} {
if _, err := os.Stat(p); os.IsNotExist(err) {
if _, err := os.Stat(p); err != nil && errors.Is(os.ErrNotExist, err) {
continue
}

6
infra/conf/conf_test.go Normal file
View File

@ -0,0 +1,6 @@
package conf_test
const (
geoipURL = "https://raw.githubusercontent.com/v2fly/geoip/release/geoip.dat"
geositeURL = "https://raw.githubusercontent.com/v2fly/domain-list-community/release/dlc.dat"
)

View File

@ -197,7 +197,7 @@ func (c *DNSConfig) Build() (*dns.Config, error) {
if len(listName) == 0 {
return nil, newError("empty geosite rule: ", domain)
}
domains, err := loadGeositeWithAttr("geosite.dat", listName)
domains, err := loadGeosite(listName)
if err != nil {
return nil, newError("failed to load geosite: ", listName).Base(err)
}

View File

@ -2,61 +2,54 @@ package conf_test
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/runtime/protoiface"
"github.com/v2fly/v2ray-core/v4/app/dns"
"github.com/v2fly/v2ray-core/v4/app/router"
"github.com/v2fly/v2ray-core/v4/common"
"github.com/v2fly/v2ray-core/v4/common/net"
"github.com/v2fly/v2ray-core/v4/common/platform"
"github.com/v2fly/v2ray-core/v4/common/platform/filesystem"
. "github.com/v2fly/v2ray-core/v4/infra/conf"
"github.com/v2fly/v2ray-core/v4/infra/conf"
)
func init() {
wd, err := os.Getwd()
common.Must(err)
if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && os.IsNotExist(err) {
common.Must(filesystem.CopyFile(platform.GetAssetLocation("geoip.dat"), filepath.Join(wd, "..", "..", "release", "config", "geoip.dat")))
tempPath := filepath.Join(wd, "..", "..", "testing", "temp")
geoipPath := filepath.Join(tempPath, "geoip.dat")
geositePath := filepath.Join(tempPath, "geosite.dat")
os.Setenv("v2ray.location.asset", tempPath)
if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && errors.Is(err, os.ErrNotExist) {
if _, err := os.Stat(geoipPath); err != nil && errors.Is(err, os.ErrNotExist) {
common.Must(os.MkdirAll(tempPath, 0755))
geoipBytes, err := common.FetchHTTPContent(geoipURL)
common.Must(err)
common.Must(filesystem.WriteFile(geoipPath, geoipBytes))
}
}
geositeFilePath := filepath.Join(wd, "geosite.dat")
os.Setenv("v2ray.location.asset", wd)
geositeFile, err := os.OpenFile(geositeFilePath, os.O_CREATE|os.O_WRONLY, 0600)
common.Must(err)
defer geositeFile.Close()
list := &router.GeoSiteList{
Entry: []*router.GeoSite{
{
CountryCode: "TEST",
Domain: []*router.Domain{
{Type: router.Domain_Full, Value: "example.com"},
},
},
},
if _, err := os.Stat(platform.GetAssetLocation("geosite.dat")); err != nil && errors.Is(err, os.ErrNotExist) {
if _, err := os.Stat(geositePath); err != nil && errors.Is(err, os.ErrNotExist) {
common.Must(os.MkdirAll(tempPath, 0755))
geositeBytes, err := common.FetchHTTPContent(geositeURL)
common.Must(err)
common.Must(filesystem.WriteFile(geositePath, geositeBytes))
}
}
listBytes, err := proto.Marshal(list)
common.Must(err)
common.Must2(geositeFile.Write(listBytes))
}
func TestDNSConfigParsing(t *testing.T) {
geositePath := platform.GetAssetLocation("geosite.dat")
defer func() {
os.Remove(geositePath)
os.Unsetenv("v2ray.location.asset")
}()
parserCreator := func() func(string) (proto.Message, error) {
return func(s string) (proto.Message, error) {
config := new(DNSConfig)
parserCreator := func() func(string) (protoiface.MessageV1, error) {
return func(s string) (protoiface.MessageV1, error) {
config := new(conf.DNSConfig)
if err := json.Unmarshal([]byte(s), config); err != nil {
return nil, err
}
@ -120,7 +113,7 @@ func TestDNSConfigParsing(t *testing.T) {
},
{
Type: dns.DomainMatchingType_Full,
Domain: "example.com",
Domain: "test.example.com",
Ip: [][]byte{{10, 0, 0, 1}},
},
{

View File

@ -242,6 +242,10 @@ func parseAttrs(attrs []string) *AttributeList {
return al
}
func loadGeosite(list string) ([]*router.Domain, error) {
return loadGeositeWithAttr("geosite.dat", list)
}
func loadGeositeWithAttr(file string, siteWithAttr string) ([]*router.Domain, error) {
parts := strings.Split(siteWithAttr, "@")
if len(parts) == 0 {
@ -288,7 +292,7 @@ func parseDomainRule(domain string) ([]*router.Domain, error) {
if len(list) == 0 {
return nil, newError("empty listname in rule: ", domain)
}
domains, err := loadGeositeWithAttr("geosite.dat", list)
domains, err := loadGeosite(list)
if err != nil {
return nil, newError("failed to load geosite: ", list).Base(err)
}

View File

@ -2,6 +2,7 @@ package conf_test
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
@ -21,11 +22,19 @@ func init() {
wd, err := os.Getwd()
common.Must(err)
if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && os.IsNotExist(err) {
common.Must(filesystem.CopyFile(platform.GetAssetLocation("geoip.dat"), filepath.Join(wd, "..", "..", "release", "config", "geoip.dat")))
}
tempPath := filepath.Join(wd, "..", "..", "testing", "temp")
geoipPath := filepath.Join(tempPath, "geoip.dat")
os.Setenv("v2ray.location.asset", wd)
os.Setenv("v2ray.location.asset", tempPath)
if _, err := os.Stat(platform.GetAssetLocation("geoip.dat")); err != nil && errors.Is(err, os.ErrNotExist) {
if _, err := os.Stat(geoipPath); err != nil && errors.Is(err, os.ErrNotExist) {
common.Must(os.MkdirAll(tempPath, 0755))
geoipBytes, err := common.FetchHTTPContent(geoipURL)
common.Must(err)
common.Must(filesystem.WriteFile(geoipPath, geoipBytes))
}
}
}
//go:linkname toCidrList github.com/v2fly/v2ray-core/v4/infra/conf.toCidrList
@ -34,7 +43,7 @@ func toCidrList(ips StringList) ([]*router.GeoIP, error)
func TestToCidrList(t *testing.T) {
t.Log(os.Getenv("v2ray.location.asset"))
common.Must(filesystem.CopyFile(platform.GetAssetLocation("geoiptestrouter.dat"), "geoip.dat"))
common.Must(filesystem.CopyFile(platform.GetAssetLocation("geoiptestrouter.dat"), platform.GetAssetLocation("geoip.dat")))
ips := StringList([]string{
"geoip:us",

Binary file not shown.

File diff suppressed because it is too large Load Diff