Initial processing and storage of logs
Signed-off-by: AKP <tom@tdpain.net>
This commit is contained in:
parent
48f63dc0db
commit
0e543f9757
18 changed files with 642 additions and 0 deletions
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
run/
|
||||
caddy_linux_amd64
|
||||
27
analytics/config/config.go
Normal file
27
analytics/config/config.go
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
package config
|
||||
|
||||
import "github.com/codemicro/analytics/analytics/config/internal/debug"
|
||||
|
||||
var Debug = debug.Enable
|
||||
|
||||
type Config struct {
|
||||
Ingest struct {
|
||||
Address string
|
||||
}
|
||||
Database struct {
|
||||
DSN string
|
||||
}
|
||||
}
|
||||
|
||||
func Load() (*Config, error) {
|
||||
cl := new(configLoader)
|
||||
if err := cl.load("config.yml"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conf := new(Config)
|
||||
conf.Ingest.Address = asString(cl.withDefault("ingest.address", "127.0.0.1:7500"))
|
||||
conf.Database.DSN = asString(cl.withDefault("database.dsn", "analytics.db"))
|
||||
|
||||
return conf, nil
|
||||
}
|
||||
17
analytics/config/internal/debug/active.go
Normal file
17
analytics/config/internal/debug/active.go
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
//go:build debug
|
||||
|
||||
package debug
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
var Enable = true
|
||||
|
||||
func init() {
|
||||
zerolog.SetGlobalLevel(zerolog.DebugLevel)
|
||||
|
||||
fmt.Println("DEBUG MODE ACTIVE")
|
||||
fmt.Println()
|
||||
}
|
||||
13
analytics/config/internal/debug/inactive.go
Normal file
13
analytics/config/internal/debug/inactive.go
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
//go:build !debug
|
||||
|
||||
package debug
|
||||
|
||||
import (
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
var Enable = false
|
||||
|
||||
func init() {
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
}
|
||||
104
analytics/config/loader.go
Normal file
104
analytics/config/loader.go
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/rs/zerolog/log"
|
||||
"gopkg.in/yaml.v3"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type configLoader struct {
|
||||
rawConfigFileContents map[string]any
|
||||
lastKey string
|
||||
}
|
||||
|
||||
func (cl *configLoader) load(fname string) error {
|
||||
cl.rawConfigFileContents = make(map[string]any)
|
||||
fcont, err := os.ReadFile(fname)
|
||||
if err != nil {
|
||||
log.Warn().Str("filename", fname).Msg("cannot load config file")
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := yaml.Unmarshal(fcont, &cl.rawConfigFileContents); err != nil {
|
||||
return fmt.Errorf("could not unmarshal config file: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type optionalItem struct {
|
||||
item any
|
||||
found bool
|
||||
}
|
||||
|
||||
var indexedPartRegexp = regexp.MustCompile(`(?m)([a-zA-Z]+)(?:\[(\d+)\])?`)
|
||||
|
||||
func (cl *configLoader) get(key string) optionalItem {
|
||||
// httpcore[2].bananas
|
||||
cl.lastKey = key
|
||||
|
||||
parts := strings.Split(key, ".")
|
||||
var cursor any = cl.rawConfigFileContents
|
||||
for _, part := range parts {
|
||||
components := indexedPartRegexp.FindStringSubmatch(part)
|
||||
key := components[1]
|
||||
index, _ := strconv.ParseInt(components[2], 10, 32)
|
||||
isIndexed := components[2] != ""
|
||||
|
||||
item, found := cursor.(map[string]any)[key]
|
||||
if !found {
|
||||
return optionalItem{nil, false}
|
||||
}
|
||||
|
||||
if isIndexed {
|
||||
arr, conversionOk := item.([]any)
|
||||
if !conversionOk {
|
||||
log.Fatal().Msgf("attempted to index non-indexable config item %s", key)
|
||||
}
|
||||
cursor = arr[index]
|
||||
} else {
|
||||
cursor = item
|
||||
}
|
||||
}
|
||||
return optionalItem{cursor, true}
|
||||
}
|
||||
|
||||
func (cl *configLoader) required(key string) optionalItem {
|
||||
opt := cl.get(key)
|
||||
if !opt.found {
|
||||
log.Fatal().Msgf("required key %s not found in config file", cl.lastKey)
|
||||
}
|
||||
return opt
|
||||
}
|
||||
|
||||
func (cl *configLoader) withDefault(key string, defaultValue any) optionalItem {
|
||||
opt := cl.get(key)
|
||||
if !opt.found {
|
||||
return optionalItem{item: defaultValue, found: true}
|
||||
}
|
||||
return opt
|
||||
}
|
||||
|
||||
func asInt(x optionalItem) int {
|
||||
if !x.found {
|
||||
return 0
|
||||
}
|
||||
return x.item.(int)
|
||||
}
|
||||
|
||||
func asString(x optionalItem) string {
|
||||
if !x.found {
|
||||
return ""
|
||||
}
|
||||
return x.item.(string)
|
||||
}
|
||||
|
||||
func asBool(x optionalItem) bool {
|
||||
if !x.found {
|
||||
return false
|
||||
}
|
||||
return x.item.(bool)
|
||||
}
|
||||
43
analytics/db/db.go
Normal file
43
analytics/db/db.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"github.com/codemicro/analytics/analytics/config"
|
||||
"github.com/codemicro/analytics/analytics/db/migrations"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/dialect/sqlitedialect"
|
||||
"github.com/uptrace/bun/driver/sqliteshim"
|
||||
"github.com/uptrace/bun/migrate"
|
||||
)
|
||||
|
||||
type DB struct {
|
||||
DB *bun.DB
|
||||
}
|
||||
|
||||
func New(conf *config.Config) (*DB, error) {
|
||||
sqldb, err := sql.Open(sqliteshim.ShimName, conf.Database.DSN)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
db := bun.NewDB(sqldb, sqlitedialect.New())
|
||||
|
||||
log.Info().Msg("migrating database")
|
||||
mig := migrate.NewMigrator(db, migrations.Migrations)
|
||||
if err := mig.Init(context.Background()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if group, err := mig.Migrate(context.Background()); err != nil {
|
||||
return nil, err
|
||||
} else if group.IsZero() {
|
||||
log.Info().Msg("no migrations to run (database is up-to-date)")
|
||||
} else {
|
||||
log.Info().Msg("migrations completed")
|
||||
}
|
||||
|
||||
return &DB{
|
||||
DB: db,
|
||||
}, nil
|
||||
}
|
||||
38
analytics/db/migrations/20230331164907_initial.go
Normal file
38
analytics/db/migrations/20230331164907_initial.go
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/codemicro/analytics/analytics/db/models"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
func init() {
|
||||
logger := log.With().Str("migration", "20230331164907").Logger()
|
||||
|
||||
tables := []any{
|
||||
&models.Request{},
|
||||
}
|
||||
|
||||
Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
|
||||
logger.Info().Msg("up")
|
||||
|
||||
for _, table := range tables {
|
||||
if _, err := db.NewCreateTable().Model(table).Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}, func(ctx context.Context, db *bun.DB) error {
|
||||
logger.Info().Msg("down")
|
||||
|
||||
for _, table := range tables {
|
||||
if _, err := db.NewDropTable().Model(table).Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
7
analytics/db/migrations/migrations.go
Normal file
7
analytics/db/migrations/migrations.go
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
package migrations
|
||||
|
||||
import (
|
||||
"github.com/uptrace/bun/migrate"
|
||||
)
|
||||
|
||||
var Migrations = migrate.NewMigrations()
|
||||
19
analytics/db/models/request.go
Normal file
19
analytics/db/models/request.go
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"github.com/uptrace/bun"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
bun.BaseModel
|
||||
|
||||
ID string `bun:",pk"`
|
||||
Time time.Time
|
||||
IPAddr string
|
||||
Host string
|
||||
RawURI string
|
||||
URI string
|
||||
Referer string
|
||||
UserAgent string
|
||||
}
|
||||
68
analytics/ingest/ingest.go
Normal file
68
analytics/ingest/ingest.go
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
package ingest
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"github.com/codemicro/analytics/analytics/config"
|
||||
"github.com/codemicro/analytics/analytics/db"
|
||||
"github.com/rs/zerolog/log"
|
||||
"io"
|
||||
"net"
|
||||
)
|
||||
|
||||
type Ingest struct {
|
||||
db *db.DB
|
||||
Listener net.Listener
|
||||
}
|
||||
|
||||
func Start(conf *config.Config, database *db.DB) (*Ingest, error) {
|
||||
ingest := &Ingest{
|
||||
db: database,
|
||||
}
|
||||
|
||||
var err error
|
||||
ingest.Listener, err = net.Listen("tcp", conf.Ingest.Address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
go ingest.serveConnections()
|
||||
|
||||
log.Info().Msgf("listener alive on %s", ingest.Listener.Addr().String())
|
||||
|
||||
return ingest, nil
|
||||
}
|
||||
|
||||
func (i *Ingest) serveConnections() {
|
||||
for {
|
||||
conn, err := i.Listener.Accept()
|
||||
if err != nil {
|
||||
if errors.Is(err, net.ErrClosed) {
|
||||
break
|
||||
}
|
||||
log.Error().Err(err).Msg("unhandled error when accepting ingest connection")
|
||||
continue
|
||||
}
|
||||
go i.processConnection(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func (i *Ingest) processConnection(conn net.Conn) {
|
||||
defer conn.Close()
|
||||
|
||||
log.Debug().Str("remote_address", conn.RemoteAddr().String()).Msg("new connection")
|
||||
|
||||
scanner := bufio.NewScanner(conn)
|
||||
for scanner.Scan() {
|
||||
i.processLog([]byte(scanner.Text()))
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
if !errors.Is(err, io.EOF) {
|
||||
log.Error().Err(err).Msg("unable to scan from connection")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
log.Debug().Str("remote_address", conn.RemoteAddr().String()).Msg("closing connection")
|
||||
}
|
||||
93
analytics/ingest/log.go
Normal file
93
analytics/ingest/log.go
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
package ingest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"github.com/codemicro/analytics/analytics/db/models"
|
||||
"github.com/lithammer/shortuuid/v4"
|
||||
"github.com/rs/zerolog/log"
|
||||
"math"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (i *Ingest) processLog(inp []byte) {
|
||||
cl := new(CaddyLog)
|
||||
if err := json.Unmarshal(inp, cl); err != nil {
|
||||
log.Warn().Err(err).Bytes("raw_input", inp).Msg("remote sending invalid JSON")
|
||||
return
|
||||
}
|
||||
|
||||
log.Debug().Msgf("got log on path %s", cl.Request.URI)
|
||||
|
||||
req, err := cl.ToRequestModel()
|
||||
if err != nil {
|
||||
log.Error().Err(err).Bytes("raw_json", inp).Msg("could not convert CaddyLog to Request")
|
||||
}
|
||||
|
||||
if _, err := i.db.DB.NewInsert().Model(req).Exec(context.Background()); err != nil {
|
||||
log.Error().Err(err).Msg("could not save request into database")
|
||||
}
|
||||
}
|
||||
|
||||
type CaddyLog struct {
|
||||
Level string `json:"level"`
|
||||
Timestamp float64 `json:"ts"`
|
||||
Logger string `json:"logger"`
|
||||
Message string `json:"msg"`
|
||||
Request struct {
|
||||
RemoteIP string `json:"remote_ip"`
|
||||
RemotePort string `json:"remote_port"`
|
||||
Protocol string `json:"proto"`
|
||||
Method string `json:"method"`
|
||||
Host string `json:"host"`
|
||||
URI string `json:"uri"`
|
||||
Headers map[string][]string `json:"headers"`
|
||||
TLS struct {
|
||||
Resumed bool `json:"resumed"`
|
||||
Version int `json:"version"`
|
||||
CipherSuite int `json:"cipher_suite"`
|
||||
Proto string `json:"proto"`
|
||||
ServerName string `json:"server_name"`
|
||||
} `json:"tls"`
|
||||
} `json:"request"`
|
||||
Duration float64 `json:"duration"`
|
||||
Size int `json:"size"`
|
||||
Status int `json:"status"`
|
||||
ResponseHeaders map[string][]string `json:"resp_headers"`
|
||||
}
|
||||
|
||||
func (cl *CaddyLog) getRequestHeader(key string) string {
|
||||
v, found := cl.Request.Headers[key]
|
||||
if !found {
|
||||
return ""
|
||||
}
|
||||
if len(v) == 0 {
|
||||
return ""
|
||||
}
|
||||
return v[0]
|
||||
}
|
||||
|
||||
func (cl *CaddyLog) ToRequestModel() (*models.Request, error) {
|
||||
parsedURL, err := url.ParseRequestURI(cl.Request.URI)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var requestTime time.Time
|
||||
{
|
||||
s, fs := math.Modf(cl.Timestamp)
|
||||
requestTime = time.Unix(int64(s), int64(fs))
|
||||
}
|
||||
|
||||
return &models.Request{
|
||||
ID: shortuuid.New(),
|
||||
Time: requestTime,
|
||||
IPAddr: cl.Request.RemoteIP,
|
||||
Host: cl.Request.Host,
|
||||
RawURI: cl.Request.URI,
|
||||
URI: parsedURL.Path,
|
||||
Referer: cl.getRequestHeader("Referer"),
|
||||
UserAgent: cl.getRequestHeader("User-Agent"),
|
||||
}, nil
|
||||
}
|
||||
47
analytics/main.go
Normal file
47
analytics/main.go
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"github.com/codemicro/analytics/analytics/config"
|
||||
"github.com/codemicro/analytics/analytics/db"
|
||||
"github.com/codemicro/analytics/analytics/ingest"
|
||||
"github.com/rs/zerolog/log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
log.Fatal().Err(err).Msg("unhandled error")
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
conf, err := config.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
database, err := db.New(conf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ig, err := ingest.Start(conf, database)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
waitForSignal(syscall.SIGINT)
|
||||
|
||||
log.Info().Msg("terminating")
|
||||
|
||||
_ = ig.Listener.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitForSignal(sig syscall.Signal) {
|
||||
cchan := make(chan os.Signal)
|
||||
signal.Notify(cchan, sig)
|
||||
<-cchan
|
||||
}
|
||||
37
go.mod
37
go.mod
|
|
@ -1,3 +1,40 @@
|
|||
module github.com/codemicro/analytics
|
||||
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/rs/zerolog v1.29.0
|
||||
github.com/uptrace/bun v1.1.12
|
||||
github.com/uptrace/bun/dialect/sqlitedialect v1.1.12
|
||||
github.com/uptrace/bun/driver/sqliteshim v1.1.12
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lithammer/shortuuid/v4 v4.0.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.16 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
golang.org/x/mod v0.8.0 // indirect
|
||||
golang.org/x/sys v0.5.0 // indirect
|
||||
golang.org/x/tools v0.6.0 // indirect
|
||||
lukechampine.com/uint128 v1.2.0 // indirect
|
||||
modernc.org/cc/v3 v3.40.0 // indirect
|
||||
modernc.org/ccgo/v3 v3.16.13 // indirect
|
||||
modernc.org/libc v1.22.2 // indirect
|
||||
modernc.org/mathutil v1.5.0 // indirect
|
||||
modernc.org/memory v1.5.0 // indirect
|
||||
modernc.org/opt v0.1.3 // indirect
|
||||
modernc.org/sqlite v1.20.4 // indirect
|
||||
modernc.org/strutil v1.1.3 // indirect
|
||||
modernc.org/token v1.1.0 // indirect
|
||||
)
|
||||
|
|
|
|||
92
go.sum
Normal file
92
go.sum
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c=
|
||||
github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y=
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w=
|
||||
github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
|
||||
github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ=
|
||||
github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0=
|
||||
github.com/uptrace/bun/dialect/sqlitedialect v1.1.12 h1:Ud31nqZmebcQpl151nb108+vtcpxJ7kfXmbPYbALBiI=
|
||||
github.com/uptrace/bun/dialect/sqlitedialect v1.1.12/go.mod h1:Pwg7s31BdF3PMBlWTnYkEn2I9ASsvatt1Ln/AERCTV4=
|
||||
github.com/uptrace/bun/driver/sqliteshim v1.1.12 h1:GMbSa7Pjjk4kjF8XURz5uMLe2PbN98e6t00sp0rx2Eo=
|
||||
github.com/uptrace/bun/driver/sqliteshim v1.1.12/go.mod h1:u67g2ewzoMDCCAqjliHAM/BJjEXfoExXlFXhx3TnXRs=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
|
||||
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
|
||||
modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
|
||||
modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
|
||||
modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
|
||||
modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
|
||||
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
|
||||
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
|
||||
modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0=
|
||||
modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug=
|
||||
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
|
||||
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
|
||||
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
|
||||
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
|
||||
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
|
||||
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
||||
modernc.org/sqlite v1.20.4 h1:J8+m2trkN+KKoE7jglyHYYYiaq5xmz2HoHJIiBlRzbE=
|
||||
modernc.org/sqlite v1.20.4/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A=
|
||||
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
|
||||
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
|
||||
modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
|
||||
7
test/Caddyfile
Normal file
7
test/Caddyfile
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
localhost {
|
||||
log {
|
||||
output net 127.0.0.1:7500
|
||||
format json
|
||||
}
|
||||
respond "Hello there!"
|
||||
}
|
||||
2
test/log.jsonl
Normal file
2
test/log.jsonl
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
{"level":"info","ts":1680281540.1000268,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"47721","proto":"HTTP/3.0","method":"GET","host":"localhost","uri":"/?balls","headers":{"Alt-Used":["localhost"],"Cookie":[],"Priority":["u=1"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],"Sec-Fetch-Site":["none"],"Sec-Fetch-Mode":["navigate"],"Pragma":["no-cache"],"Accept-Language":["en-GB,en;q=0.5"],"Accept-Encoding":["gzip, deflate, br"],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-User":["?1"],"Cache-Control":["no-cache"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h3","server_name":"localhost"}},"user_id":"","duration":0.000019051,"size":12,"status":200,"resp_headers":{"Server":["Caddy"],"Content-Type":["text/plain; charset=utf-8"]}}
|
||||
{"level":"info","ts":1680281540.3660147,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"47721","proto":"HTTP/3.0","method":"GET","host":"localhost","uri":"/favicon.ico","headers":{"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0"],"Accept-Language":["en-GB,en;q=0.5"],"Cookie":[],"Sec-Fetch-Dest":["image"],"Sec-Fetch-Mode":["no-cors"],"Cache-Control":["no-cache"],"Priority":["u=6"],"Accept":["image/avif,image/webp,*/*"],"Accept-Encoding":["gzip, deflate, br"],"Alt-Used":["localhost"],"Referer":["https://localhost/?balls"],"Sec-Fetch-Site":["same-origin"],"Pragma":["no-cache"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h3","server_name":"localhost"}},"user_id":"","duration":0.000061116,"size":12,"status":200,"resp_headers":{"Server":["Caddy"],"Content-Type":["text/plain; charset=utf-8"]}}
|
||||
2
test/logs2.jsonl
Normal file
2
test/logs2.jsonl
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
{"level":"info","ts":1680281834.4265301,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"44882","proto":"HTTP/3.0","method":"GET","host":"localhost","uri":"/?balls","headers":{"Accept-Language":["en-GB,en;q=0.5"],"Alt-Used":["localhost"],"Cookie":[],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-Mode":["navigate"],"Pragma":["no-cache"],"Priority":["u=1"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],"Accept-Encoding":["gzip, deflate, br"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Site":["cross-site"],"Cache-Control":["no-cache"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h3","server_name":"localhost"}},"user_id":"","duration":0.000022127,"size":12,"status":200,"resp_headers":{"Server":["Caddy"],"Content-Type":["text/plain; charset=utf-8"]}}
|
||||
{"level":"info","ts":1680281834.5157068,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"44882","proto":"HTTP/3.0","method":"GET","host":"localhost","uri":"/favicon.ico","headers":{"Referer":["https://localhost/?balls"],"Cookie":[],"Sec-Fetch-Dest":["image"],"Sec-Fetch-Site":["same-origin"],"Pragma":["no-cache"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0"],"Accept":["image/avif,image/webp,*/*"],"Accept-Encoding":["gzip, deflate, br"],"Cache-Control":["no-cache"],"Sec-Fetch-Mode":["no-cors"],"Priority":["u=6"],"Accept-Language":["en-GB,en;q=0.5"],"Alt-Used":["localhost"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h3","server_name":"localhost"}},"user_id":"","duration":0.000020068,"size":12,"status":200,"resp_headers":{"Content-Type":["text/plain; charset=utf-8"],"Server":["Caddy"]}}
|
||||
24
test/sinkhole.py
Normal file
24
test/sinkhole.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import socket
|
||||
import sys
|
||||
|
||||
SOCKET=7502
|
||||
|
||||
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
serversocket.bind(('localhost', SOCKET))
|
||||
serversocket.listen(5) # become a server socket, maximum 5 connections
|
||||
|
||||
print(f"[*] Alive at localhost:{SOCKET}", file=sys.stderr)
|
||||
|
||||
while True:
|
||||
connection, address = serversocket.accept()
|
||||
print(f"[+] New connection {address=}", file=sys.stderr)
|
||||
while True:
|
||||
buf = connection.recv(64)
|
||||
if len(buf) > 0:
|
||||
print(buf)
|
||||
else:
|
||||
connection.close()
|
||||
print(f"[-] Connection closed", file=sys.stderr)
|
||||
break
|
||||
Reference in a new issue