Skip to content

Commit 65cad0a

Browse files
committedSep 16, 2020
add geoIP database support: from local file or from url
1 parent 0bb71b6 commit 65cad0a

11 files changed

+250
-40
lines changed
 

‎.gitignore

+7
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,19 @@
55
*.so
66
*.dylib
77
*.html
8+
*.zip
9+
*.tar
10+
*.tar.gz
811

912
/auth_exporter
1013
/auth_exporter.service
1114

15+
/GeoLite*
16+
1217
/dist
1318

19+
/.vscode
20+
1421
# Test binary, built with `go test -c`
1522
*.test
1623

‎README.md

+7-3
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ make build
5555
./auth_exporter <flags>
5656
```
5757

58-
By default, metrics will be collecting from `/var/log/auth.log` and will be available at http://localhost:9090/metrics. This means that the user who runs `auth_exporter` should have read permission to file `/var/log/auth.log`. You can changed logfile location, port and endpoint by using the`-auth.log`, `-port` and `-endpoint` flags.
58+
By default, metrics will be collecting from `/var/log/auth.log` and will be available at http://localhost:9090/metrics. This means that the user who runs `auth_exporter` should have read permission to file `/var/log/auth.log`. You can changed logfile location, port and endpoint by using the`-auth.log`, `-prom.port` and `-prom.endpoint` flags.
5959

6060
Available configuration flags:
6161

@@ -65,9 +65,13 @@ Available configuration flags:
6565
Usage of ./auth_exporter:
6666
-auth.log string
6767
Path to auth.log (default "/var/log/auth.log")
68-
-endpoint string
68+
-geo.db string
69+
Path to geoIP database file
70+
-geo.lang string
71+
Output language format (default "en")
72+
-prom.endpoint string
6973
Endpoint used for metrics (default "/metrics")
70-
-port string
74+
-prom.port string
7175
Port for prometheus metrics to listen on (default "9991")
7276
```
7377

‎auth_exporter.go

+10-12
Original file line numberDiff line numberDiff line change
@@ -12,38 +12,36 @@ import (
1212
)
1313

1414
var (
15-
promPort = flag.String("port", "9991", "Port for prometheus metrics to listen on")
16-
promPath = flag.String("endpoint", "/metrics", "Endpoint used for metrics")
15+
promPort = flag.String("prom.port", "9991", "Port for prometheus metrics to listen on")
16+
promPath = flag.String("prom.endpoint", "/metrics", "Endpoint used for metrics")
1717
authlogPath = flag.String("auth.log", "/var/log/auth.log", "Path to auth.log")
18+
geodbPath = flag.String("geo.db", "", "Path to geoIP database file")
19+
geodbLang = flag.String("geo.lang", "en", "Output language format")
20+
geodbURL = flag.String("geo.url", "https://freegeoip.live/json/", "URL for geoIP database API ")
21+
geodbType = flag.String("geo.type", "", "Type of geoIP database: db, url")
1822
version = "development"
1923
)
2024

2125
func main() {
22-
2326
// Load command line arguments
2427
flag.Parse()
25-
2628
// Setup signal catching
2729
sigs := make(chan os.Signal, 1)
28-
2930
// Catch listed signals
3031
signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
31-
3232
// Method invoked upon seeing signal
3333
go func() {
3434
s := <-sigs
35-
log.Printf("RECEIVED SIGNAL: %s", s)
36-
log.Printf("Stopping : %s", filepath.Base(os.Args[0]))
35+
log.Printf("RECEIVED SIGNAL %s", s)
36+
log.Printf("Stopping %s", filepath.Base(os.Args[0]))
3737
os.Exit(0)
3838
}()
39-
4039
log.Printf("Starting %s", filepath.Base(os.Args[0]))
41-
log.Printf("Version: %s", version)
42-
40+
log.Printf("Version %s", version)
4341
// Setup parameters for exporter
4442
promexporter.SetPromPortandPath(*promPort, *promPath)
4543
promexporter.SetAuthlogPath(*authlogPath)
46-
44+
promexporter.SetGeodbPath(*geodbType, *geodbPath, *geodbLang, *geodbURL)
4745
// Start exporter
4846
promexporter.Start()
4947
}

‎promexporter/endpoint.go

+3-5
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,18 @@ import (
88
)
99

1010
var (
11-
promPort = "9991"
12-
promEndpoint = "/metrics"
11+
promPort string
12+
promEndpoint string
1313
)
1414

1515
// SetPromPortandPath sets HTTP endpoint parameters from command line arguments 'port' and 'endpoint'
1616
func SetPromPortandPath(port, endpoint string) {
1717
promPort = port
1818
promEndpoint = endpoint
19+
log.Printf("Use port %s and HTTP endpoint %s", promPort, promEndpoint)
1920
}
2021

2122
func startPromEndpoint() {
22-
23-
log.Printf("Use port: %s and HTTP endpoint: %s", promPort, promEndpoint)
24-
2523
go func() {
2624
http.Handle(promEndpoint, promhttp.Handler())
2725
log.Fatalln(http.ListenAndServe(":"+promPort, nil))

‎promexporter/endpoint_test.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,7 @@ func TestSetPromPortandPath(t *testing.T) {
1010
testEndpoit = "/metrics"
1111
)
1212
SetPromPortandPath(testPort, testEndpoit)
13-
1413
if testPort != promPort || testEndpoit != promEndpoint {
15-
t.Errorf("Variables do not match: %s, want: %s; %s, want: %s", testPort, promPort, testEndpoit, promEndpoint)
14+
t.Errorf("\nVariables do not match: %s,\nwant: %s;\nendpoint: %s,\nwant: %s", testPort, promPort, testEndpoit, promEndpoint)
1615
}
1716
}

‎promexporter/exporter.go

+2-5
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,16 @@ import (
77
)
88

99
var (
10-
authlogPath = "/var/log/auth.log"
10+
authlogPath string
1111
)
1212

1313
// SetAuthlogPath sets path for 'auth.log' from command line argument 'auth.log'
1414
func SetAuthlogPath(filePath string) {
1515
authlogPath = filePath
16+
log.Printf("Log for parsing %s", authlogPath)
1617
}
1718

1819
func startParserAuthlog(filePath string) {
19-
20-
log.Printf("Log for parsing: %s", authlogPath)
21-
2220
t, err := tail.TailFile(filePath, tail.Config{
2321
Follow: true,
2422
ReOpen: true,
@@ -29,7 +27,6 @@ func startParserAuthlog(filePath string) {
2927
for line := range t.Lines {
3028
parseLine(line)
3129
}
32-
3330
}
3431

3532
// Start runs promhttp endpoind and parsing log process

‎promexporter/exporter_test.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@ func TestSetAuthlogPath(t *testing.T) {
99
testLog = "/test_log/auth.log"
1010
)
1111
SetAuthlogPath(testLog)
12-
1312
if testLog != authlogPath {
14-
t.Errorf("Variables do not match: %s, want: %s", testLog, authlogPath)
13+
t.Errorf("Variables do not match: %s,\nwant: %s", testLog, authlogPath)
1514
}
1615
}

‎promexporter/geo.go

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
package promexporter
2+
3+
import (
4+
"encoding/json"
5+
"io/ioutil"
6+
"log"
7+
"net"
8+
"net/http"
9+
10+
"github.com/oschwald/geoip2-golang"
11+
)
12+
13+
var (
14+
geodbType string
15+
geodbPath string
16+
geoLang string
17+
geoURL string
18+
geodbIs = false
19+
)
20+
21+
type geoInfo struct {
22+
countyISOCode string
23+
countryName string
24+
cityName string
25+
}
26+
27+
// SetGeodbPath sets geoIP database parameters from command line argument geo.type, geo.db and geo.lang or geo.url
28+
func SetGeodbPath(geoType, filePath, outputLang, url string) {
29+
geodbType = geoType
30+
geodbPath = filePath
31+
geoLang = outputLang
32+
geoURL = url
33+
checkGeoDBFlags()
34+
}
35+
36+
func checkGeoDBFlags() {
37+
notSetLogMsg := "GeoIP database is not set and not use"
38+
switch geodbType {
39+
case "":
40+
log.Println(notSetLogMsg)
41+
case "db":
42+
if geodbPath == "" {
43+
log.Println("Error geo.db flag is not set", geodbPath)
44+
log.Println(notSetLogMsg)
45+
} else {
46+
geodbIs = true
47+
log.Println("Use GeoIp database file", geodbPath)
48+
}
49+
case "url":
50+
geodbIs = true
51+
log.Println("Use GeoIp database url", geoURL)
52+
default:
53+
log.Println("Error geo.type flag value is incorect", geodbType)
54+
log.Println(notSetLogMsg)
55+
}
56+
}
57+
58+
func getIPDetailsFromLocalDB(returnValues *geoInfo, ipAddres string) {
59+
geodb, err := geoip2.Open(geodbPath)
60+
if err != nil {
61+
log.Println("Error opening GeoIp database file", err)
62+
return
63+
}
64+
defer geodb.Close()
65+
ip := net.ParseIP(ipAddres)
66+
if ip == nil {
67+
log.Println("Error parsing ip address", ipAddres)
68+
return
69+
}
70+
record, err := geodb.City(ip)
71+
if err != nil {
72+
log.Println("Error getting location details", err)
73+
return
74+
}
75+
returnValues.countyISOCode = record.Country.IsoCode
76+
returnValues.countryName = record.Country.Names[geoLang]
77+
returnValues.cityName = record.City.Names[geoLang]
78+
}
79+
80+
func getIPDetailsFromURL(returnValues *geoInfo, ipAddres string) {
81+
response, err := http.Get(geoURL + ipAddres)
82+
if err != nil {
83+
log.Println("Error getting GeoIp URL", err)
84+
return
85+
}
86+
defer response.Body.Close()
87+
body, err := ioutil.ReadAll(response.Body)
88+
if err != nil {
89+
log.Println("Error getting body from GeoIp URL", err)
90+
return
91+
}
92+
var parseData map[string]interface{}
93+
err = json.Unmarshal([]byte(body), &parseData)
94+
if err != nil {
95+
log.Println("Error parsing json-encoded body from GeoIp URL", err)
96+
return
97+
}
98+
returnValues.countyISOCode = getMap(parseData, "country_code")
99+
returnValues.countryName = getMap(parseData, "country_name")
100+
returnValues.cityName = getMap(parseData, "city")
101+
}
102+
103+
func getMap(data map[string]interface{}, key string) string {
104+
str, ok := data[key].(string)
105+
if !ok {
106+
log.Printf("Error for key %s value %s is not a string", key, str)
107+
}
108+
return str
109+
}

‎promexporter/geo_test.go

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package promexporter
2+
3+
import (
4+
"bytes"
5+
"log"
6+
"os"
7+
"strings"
8+
"testing"
9+
)
10+
11+
func TestSetGeodbPath(t *testing.T) {
12+
type args struct {
13+
geoType string
14+
filePath string
15+
outputLang string
16+
url string
17+
}
18+
tests := []struct {
19+
name string
20+
args args
21+
want string
22+
}{
23+
{"defaultGeodbType",
24+
args{"", "", "", ""},
25+
"GeoIP database is not set and not use",
26+
},
27+
{"dbGeodbTypePathEmpty",
28+
args{"db", "", "", ""},
29+
"Error geo.db flag is not set",
30+
},
31+
{"dbGeodbTypePathNotEmpty",
32+
args{"db", "test.file", "", ""},
33+
"Use GeoIp database file",
34+
},
35+
{"urlGeodbType",
36+
args{"url", "", "", "http://test"},
37+
"Use GeoIp database url",
38+
},
39+
{"badGeodbType",
40+
args{"test", "", "", ""},
41+
"Error geo.type flag value is incorect",
42+
},
43+
}
44+
for _, tt := range tests {
45+
t.Run(tt.name, func(t *testing.T) {
46+
var buf bytes.Buffer
47+
log.SetOutput(&buf)
48+
defer func() {
49+
log.SetOutput(os.Stdout)
50+
}()
51+
SetGeodbPath(tt.args.geoType, tt.args.filePath, tt.args.outputLang, tt.args.url)
52+
if got := buf.String(); !strings.Contains(got, tt.want) {
53+
t.Errorf("\nSetGeodbPath() log output:\n%s\nnot containt want:\n%s", got, tt.want)
54+
}
55+
})
56+
}
57+
}
58+
59+
func TestGetMap(t *testing.T) {
60+
type args struct {
61+
data map[string]interface{}
62+
key string
63+
}
64+
tests := []struct {
65+
name string
66+
args args
67+
want string
68+
}{
69+
{"existKey",
70+
args{
71+
map[string]interface{}{"country_code": "US"},
72+
"country_code",
73+
},
74+
"US"},
75+
{"notExistKey",
76+
args{
77+
map[string]interface{}{"country_code": "US"},
78+
"city",
79+
},
80+
""},
81+
}
82+
for _, tt := range tests {
83+
t.Run(tt.name, func(t *testing.T) {
84+
if got := getMap(tt.args.data, tt.args.key); got != tt.want {
85+
t.Errorf("\ngetMap() =\n%v,\nwant=\n%v", got, tt.want)
86+
}
87+
})
88+
}
89+
}

‎promexporter/parser.go

+19-8
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@ import (
99
)
1010

1111
var (
12-
authLinePrefix = "^(?P<date>[A-Z][a-z]{2}\\s+\\d{1,2}) (?P<time>(\\d{2}:?){3}) (?P<host>[a-zA-Z0-9_\\-\\.]+) (?P<ident>[a-zA-Z0-9_\\-]+)(\\[(?P<pid>\\d+)\\])?: "
13-
12+
authLinePrefix = "^(?P<date>[A-Z][a-z]{2}\\s+\\d{1,2}) (?P<time>(\\d{2}:?){3}) (?P<host>[a-zA-Z0-9_\\-\\.]+) (?P<ident>[a-zA-Z0-9_\\-]+)(\\[(?P<pid>\\d+)\\])?: "
1413
authLineRegexps = map[string]*regexp.Regexp{
1514
"authAccepted": regexp.MustCompile(authLinePrefix + "Accepted (password|publickey) for (?P<user>.*) from (?P<ipAddress>.*) port"),
1615
"authFailed": regexp.MustCompile(authLinePrefix + "Failed (password|publickey) for (invalid user )?(?P<user>.*) from (?P<ipAddress>.*) port"),
@@ -22,7 +21,7 @@ var (
2221
Name: "auth_exporter_auth_events",
2322
Help: "The total number of auth events by user and IP addresses",
2423
},
25-
[]string{"eventType", "user", "ipAddress"})
24+
[]string{"eventType", "user", "ipAddress", "countyISOCode", "countryName", "cityName"})
2625
)
2726

2827
type authLogLine struct {
@@ -33,32 +32,44 @@ type authLogLine struct {
3332

3433
func parseLine(line *tail.Line) {
3534
parsedLog := &authLogLine{}
36-
3735
matches := make(map[string]string)
38-
3936
// Find the type of log and parse it
4037
for t, re := range authLineRegexps {
4138
if re.MatchString(line.Text) {
4239
parsedLog.Type = t
4340
matches = getMatches(line.Text, re)
44-
//fmt.Println(matches)
4541
continue
4642
}
4743
}
4844
// Skip if not matching
4945
if len(matches) == 0 {
5046
return
5147
}
48+
geoIPData := &geoInfo{}
5249
parsedLog.Username = matches["user"]
5350
parsedLog.IPAddress = matches["ipAddress"]
51+
// Get geo information
52+
if geodbIs {
53+
if geodbType == "db" {
54+
getIPDetailsFromLocalDB(geoIPData, parsedLog.IPAddress)
55+
} else {
56+
getIPDetailsFromURL(geoIPData, parsedLog.IPAddress)
57+
}
58+
}
5459
// Add metric
55-
authVentsMetric.WithLabelValues(parsedLog.Type, parsedLog.Username, parsedLog.IPAddress).Inc()
60+
authVentsMetric.WithLabelValues(
61+
parsedLog.Type,
62+
parsedLog.Username,
63+
parsedLog.IPAddress,
64+
geoIPData.countyISOCode,
65+
geoIPData.countryName,
66+
geoIPData.cityName,
67+
).Inc()
5668
}
5769

5870
func getMatches(line string, re *regexp.Regexp) map[string]string {
5971
matches := re.FindStringSubmatch(line)
6072
results := make(map[string]string)
61-
6273
// Get the basic information out of the log
6374
for i, name := range re.SubexpNames() {
6475
if i != 0 && len(matches) > i {

‎promexporter/parser_test.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"testing"
77
)
88

9-
func Test_getMatches(t *testing.T) {
9+
func TestGetMatches(t *testing.T) {
1010
type args struct {
1111
line string
1212
re *regexp.Regexp
@@ -113,11 +113,10 @@ func Test_getMatches(t *testing.T) {
113113
},
114114
},
115115
}
116-
117116
for _, tt := range tests {
118117
t.Run(tt.name, func(t *testing.T) {
119118
if got := getMatches(tt.args.line, tt.args.re); !reflect.DeepEqual(got, tt.want) {
120-
t.Errorf("getMatches() = %v, want %v", got, tt.want)
119+
t.Errorf("\ngetMatches():\n%v,\nwant:\n%v", got, tt.want)
121120
}
122121
})
123122
}

0 commit comments

Comments
 (0)
Please sign in to comment.