diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9d60ad1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +run/ +caddy_linux_amd64 diff --git a/analytics/config/config.go b/analytics/config/config.go new file mode 100644 index 0000000..591431c --- /dev/null +++ b/analytics/config/config.go @@ -0,0 +1,27 @@ +package config + +import "github.com/codemicro/analytics/analytics/config/internal/debug" + +var Debug = debug.Enable + +type Config struct { + Ingest struct { + Address string + } + Database struct { + DSN string + } +} + +func Load() (*Config, error) { + cl := new(configLoader) + if err := cl.load("config.yml"); err != nil { + return nil, err + } + + conf := new(Config) + conf.Ingest.Address = asString(cl.withDefault("ingest.address", "127.0.0.1:7500")) + conf.Database.DSN = asString(cl.withDefault("database.dsn", "analytics.db")) + + return conf, nil +} diff --git a/analytics/config/internal/debug/active.go b/analytics/config/internal/debug/active.go new file mode 100644 index 0000000..4df8113 --- /dev/null +++ b/analytics/config/internal/debug/active.go @@ -0,0 +1,17 @@ +//go:build debug + +package debug + +import ( + "fmt" + "github.com/rs/zerolog" +) + +var Enable = true + +func init() { + zerolog.SetGlobalLevel(zerolog.DebugLevel) + + fmt.Println("DEBUG MODE ACTIVE") + fmt.Println() +} diff --git a/analytics/config/internal/debug/inactive.go b/analytics/config/internal/debug/inactive.go new file mode 100644 index 0000000..a4b7800 --- /dev/null +++ b/analytics/config/internal/debug/inactive.go @@ -0,0 +1,13 @@ +//go:build !debug + +package debug + +import ( + "github.com/rs/zerolog" +) + +var Enable = false + +func init() { + zerolog.SetGlobalLevel(zerolog.InfoLevel) +} diff --git a/analytics/config/loader.go b/analytics/config/loader.go new file mode 100644 index 0000000..384372a --- /dev/null +++ b/analytics/config/loader.go @@ -0,0 +1,104 @@ +package config + +import ( + "fmt" + "github.com/rs/zerolog/log" + "gopkg.in/yaml.v3" + "os" + "regexp" + "strconv" + "strings" +) + +type configLoader struct { + rawConfigFileContents map[string]any + lastKey string +} + +func (cl *configLoader) load(fname string) error { + cl.rawConfigFileContents = make(map[string]any) + fcont, err := os.ReadFile(fname) + if err != nil { + log.Warn().Str("filename", fname).Msg("cannot load config file") + return nil + } + + if err := yaml.Unmarshal(fcont, &cl.rawConfigFileContents); err != nil { + return fmt.Errorf("could not unmarshal config file: %v", err) + } + return nil +} + +type optionalItem struct { + item any + found bool +} + +var indexedPartRegexp = regexp.MustCompile(`(?m)([a-zA-Z]+)(?:\[(\d+)\])?`) + +func (cl *configLoader) get(key string) optionalItem { + // httpcore[2].bananas + cl.lastKey = key + + parts := strings.Split(key, ".") + var cursor any = cl.rawConfigFileContents + for _, part := range parts { + components := indexedPartRegexp.FindStringSubmatch(part) + key := components[1] + index, _ := strconv.ParseInt(components[2], 10, 32) + isIndexed := components[2] != "" + + item, found := cursor.(map[string]any)[key] + if !found { + return optionalItem{nil, false} + } + + if isIndexed { + arr, conversionOk := item.([]any) + if !conversionOk { + log.Fatal().Msgf("attempted to index non-indexable config item %s", key) + } + cursor = arr[index] + } else { + cursor = item + } + } + return optionalItem{cursor, true} +} + +func (cl *configLoader) required(key string) optionalItem { + opt := cl.get(key) + if !opt.found { + log.Fatal().Msgf("required key %s not found in config file", cl.lastKey) + } + return opt +} + +func (cl *configLoader) withDefault(key string, defaultValue any) optionalItem { + opt := cl.get(key) + if !opt.found { + return optionalItem{item: defaultValue, found: true} + } + return opt +} + +func asInt(x optionalItem) int { + if !x.found { + return 0 + } + return x.item.(int) +} + +func asString(x optionalItem) string { + if !x.found { + return "" + } + return x.item.(string) +} + +func asBool(x optionalItem) bool { + if !x.found { + return false + } + return x.item.(bool) +} diff --git a/analytics/db/db.go b/analytics/db/db.go new file mode 100644 index 0000000..4c89507 --- /dev/null +++ b/analytics/db/db.go @@ -0,0 +1,43 @@ +package db + +import ( + "context" + "database/sql" + "github.com/codemicro/analytics/analytics/config" + "github.com/codemicro/analytics/analytics/db/migrations" + "github.com/rs/zerolog/log" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/sqlitedialect" + "github.com/uptrace/bun/driver/sqliteshim" + "github.com/uptrace/bun/migrate" +) + +type DB struct { + DB *bun.DB +} + +func New(conf *config.Config) (*DB, error) { + sqldb, err := sql.Open(sqliteshim.ShimName, conf.Database.DSN) + if err != nil { + panic(err) + } + + db := bun.NewDB(sqldb, sqlitedialect.New()) + + log.Info().Msg("migrating database") + mig := migrate.NewMigrator(db, migrations.Migrations) + if err := mig.Init(context.Background()); err != nil { + return nil, err + } + if group, err := mig.Migrate(context.Background()); err != nil { + return nil, err + } else if group.IsZero() { + log.Info().Msg("no migrations to run (database is up-to-date)") + } else { + log.Info().Msg("migrations completed") + } + + return &DB{ + DB: db, + }, nil +} diff --git a/analytics/db/migrations/20230331164907_initial.go b/analytics/db/migrations/20230331164907_initial.go new file mode 100644 index 0000000..7d1deb8 --- /dev/null +++ b/analytics/db/migrations/20230331164907_initial.go @@ -0,0 +1,38 @@ +package migrations + +import ( + "context" + "github.com/codemicro/analytics/analytics/db/models" + "github.com/rs/zerolog/log" + "github.com/uptrace/bun" +) + +func init() { + logger := log.With().Str("migration", "20230331164907").Logger() + + tables := []any{ + &models.Request{}, + } + + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + logger.Info().Msg("up") + + for _, table := range tables { + if _, err := db.NewCreateTable().Model(table).Exec(ctx); err != nil { + return err + } + } + + return nil + }, func(ctx context.Context, db *bun.DB) error { + logger.Info().Msg("down") + + for _, table := range tables { + if _, err := db.NewDropTable().Model(table).Exec(ctx); err != nil { + return err + } + } + + return nil + }) +} diff --git a/analytics/db/migrations/migrations.go b/analytics/db/migrations/migrations.go new file mode 100644 index 0000000..f7346fb --- /dev/null +++ b/analytics/db/migrations/migrations.go @@ -0,0 +1,7 @@ +package migrations + +import ( + "github.com/uptrace/bun/migrate" +) + +var Migrations = migrate.NewMigrations() diff --git a/analytics/db/models/request.go b/analytics/db/models/request.go new file mode 100644 index 0000000..6f5f49e --- /dev/null +++ b/analytics/db/models/request.go @@ -0,0 +1,19 @@ +package models + +import ( + "github.com/uptrace/bun" + "time" +) + +type Request struct { + bun.BaseModel + + ID string `bun:",pk"` + Time time.Time + IPAddr string + Host string + RawURI string + URI string + Referer string + UserAgent string +} diff --git a/analytics/ingest/ingest.go b/analytics/ingest/ingest.go new file mode 100644 index 0000000..c2e7198 --- /dev/null +++ b/analytics/ingest/ingest.go @@ -0,0 +1,68 @@ +package ingest + +import ( + "bufio" + "errors" + "github.com/codemicro/analytics/analytics/config" + "github.com/codemicro/analytics/analytics/db" + "github.com/rs/zerolog/log" + "io" + "net" +) + +type Ingest struct { + db *db.DB + Listener net.Listener +} + +func Start(conf *config.Config, database *db.DB) (*Ingest, error) { + ingest := &Ingest{ + db: database, + } + + var err error + ingest.Listener, err = net.Listen("tcp", conf.Ingest.Address) + if err != nil { + return nil, err + } + + go ingest.serveConnections() + + log.Info().Msgf("listener alive on %s", ingest.Listener.Addr().String()) + + return ingest, nil +} + +func (i *Ingest) serveConnections() { + for { + conn, err := i.Listener.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) { + break + } + log.Error().Err(err).Msg("unhandled error when accepting ingest connection") + continue + } + go i.processConnection(conn) + } +} + +func (i *Ingest) processConnection(conn net.Conn) { + defer conn.Close() + + log.Debug().Str("remote_address", conn.RemoteAddr().String()).Msg("new connection") + + scanner := bufio.NewScanner(conn) + for scanner.Scan() { + i.processLog([]byte(scanner.Text())) + } + + if err := scanner.Err(); err != nil { + if !errors.Is(err, io.EOF) { + log.Error().Err(err).Msg("unable to scan from connection") + return + } + } + + log.Debug().Str("remote_address", conn.RemoteAddr().String()).Msg("closing connection") +} diff --git a/analytics/ingest/log.go b/analytics/ingest/log.go new file mode 100644 index 0000000..e79cacc --- /dev/null +++ b/analytics/ingest/log.go @@ -0,0 +1,93 @@ +package ingest + +import ( + "context" + "encoding/json" + "github.com/codemicro/analytics/analytics/db/models" + "github.com/lithammer/shortuuid/v4" + "github.com/rs/zerolog/log" + "math" + "net/url" + "time" +) + +func (i *Ingest) processLog(inp []byte) { + cl := new(CaddyLog) + if err := json.Unmarshal(inp, cl); err != nil { + log.Warn().Err(err).Bytes("raw_input", inp).Msg("remote sending invalid JSON") + return + } + + log.Debug().Msgf("got log on path %s", cl.Request.URI) + + req, err := cl.ToRequestModel() + if err != nil { + log.Error().Err(err).Bytes("raw_json", inp).Msg("could not convert CaddyLog to Request") + } + + if _, err := i.db.DB.NewInsert().Model(req).Exec(context.Background()); err != nil { + log.Error().Err(err).Msg("could not save request into database") + } +} + +type CaddyLog struct { + Level string `json:"level"` + Timestamp float64 `json:"ts"` + Logger string `json:"logger"` + Message string `json:"msg"` + Request struct { + RemoteIP string `json:"remote_ip"` + RemotePort string `json:"remote_port"` + Protocol string `json:"proto"` + Method string `json:"method"` + Host string `json:"host"` + URI string `json:"uri"` + Headers map[string][]string `json:"headers"` + TLS struct { + Resumed bool `json:"resumed"` + Version int `json:"version"` + CipherSuite int `json:"cipher_suite"` + Proto string `json:"proto"` + ServerName string `json:"server_name"` + } `json:"tls"` + } `json:"request"` + Duration float64 `json:"duration"` + Size int `json:"size"` + Status int `json:"status"` + ResponseHeaders map[string][]string `json:"resp_headers"` +} + +func (cl *CaddyLog) getRequestHeader(key string) string { + v, found := cl.Request.Headers[key] + if !found { + return "" + } + if len(v) == 0 { + return "" + } + return v[0] +} + +func (cl *CaddyLog) ToRequestModel() (*models.Request, error) { + parsedURL, err := url.ParseRequestURI(cl.Request.URI) + if err != nil { + return nil, err + } + + var requestTime time.Time + { + s, fs := math.Modf(cl.Timestamp) + requestTime = time.Unix(int64(s), int64(fs)) + } + + return &models.Request{ + ID: shortuuid.New(), + Time: requestTime, + IPAddr: cl.Request.RemoteIP, + Host: cl.Request.Host, + RawURI: cl.Request.URI, + URI: parsedURL.Path, + Referer: cl.getRequestHeader("Referer"), + UserAgent: cl.getRequestHeader("User-Agent"), + }, nil +} diff --git a/analytics/main.go b/analytics/main.go new file mode 100644 index 0000000..9963f1d --- /dev/null +++ b/analytics/main.go @@ -0,0 +1,47 @@ +package main + +import ( + "github.com/codemicro/analytics/analytics/config" + "github.com/codemicro/analytics/analytics/db" + "github.com/codemicro/analytics/analytics/ingest" + "github.com/rs/zerolog/log" + "os" + "os/signal" + "syscall" +) + +func main() { + if err := run(); err != nil { + log.Fatal().Err(err).Msg("unhandled error") + } +} + +func run() error { + conf, err := config.Load() + if err != nil { + return err + } + + database, err := db.New(conf) + if err != nil { + return err + } + + ig, err := ingest.Start(conf, database) + if err != nil { + return err + } + + waitForSignal(syscall.SIGINT) + + log.Info().Msg("terminating") + + _ = ig.Listener.Close() + return nil +} + +func waitForSignal(sig syscall.Signal) { + cchan := make(chan os.Signal) + signal.Notify(cchan, sig) + <-cchan +} diff --git a/go.mod b/go.mod index fc61250..2f205f0 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,40 @@ module github.com/codemicro/analytics go 1.20 + +require ( + github.com/rs/zerolog v1.29.0 + github.com/uptrace/bun v1.1.12 + github.com/uptrace/bun/dialect/sqlitedialect v1.1.12 + github.com/uptrace/bun/driver/sqliteshim v1.1.12 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/lithammer/shortuuid/v4 v4.0.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-sqlite3 v1.14.16 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/tools v0.6.0 // indirect + lukechampine.com/uint128 v1.2.0 // indirect + modernc.org/cc/v3 v3.40.0 // indirect + modernc.org/ccgo/v3 v3.16.13 // indirect + modernc.org/libc v1.22.2 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/opt v0.1.3 // indirect + modernc.org/sqlite v1.20.4 // indirect + modernc.org/strutil v1.1.3 // indirect + modernc.org/token v1.1.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..35c5e40 --- /dev/null +++ b/go.sum @@ -0,0 +1,92 @@ +github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lithammer/shortuuid/v4 v4.0.0 h1:QRbbVkfgNippHOS8PXDkti4NaWeyYfcBTHtw7k08o4c= +github.com/lithammer/shortuuid/v4 v4.0.0/go.mod h1:Zs8puNcrvf2rV9rTH51ZLLcj7ZXqQI3lv67aw4KiB1Y= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.29.0 h1:Zes4hju04hjbvkVkOhdl2HpZa+0PmVwigmo8XoORE5w= +github.com/rs/zerolog v1.29.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= +github.com/uptrace/bun v1.1.12 h1:sOjDVHxNTuM6dNGaba0wUuz7KvDE1BmNu9Gqs2gJSXQ= +github.com/uptrace/bun v1.1.12/go.mod h1:NPG6JGULBeQ9IU6yHp7YGELRa5Agmd7ATZdz4tGZ6z0= +github.com/uptrace/bun/dialect/sqlitedialect v1.1.12 h1:Ud31nqZmebcQpl151nb108+vtcpxJ7kfXmbPYbALBiI= +github.com/uptrace/bun/dialect/sqlitedialect v1.1.12/go.mod h1:Pwg7s31BdF3PMBlWTnYkEn2I9ASsvatt1Ln/AERCTV4= +github.com/uptrace/bun/driver/sqliteshim v1.1.12 h1:GMbSa7Pjjk4kjF8XURz5uMLe2PbN98e6t00sp0rx2Eo= +github.com/uptrace/bun/driver/sqliteshim v1.1.12/go.mod h1:u67g2ewzoMDCCAqjliHAM/BJjEXfoExXlFXhx3TnXRs= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.20.4 h1:J8+m2trkN+KKoE7jglyHYYYiaq5xmz2HoHJIiBlRzbE= +modernc.org/sqlite v1.20.4/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A= +modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE= diff --git a/test/Caddyfile b/test/Caddyfile new file mode 100644 index 0000000..1f1b563 --- /dev/null +++ b/test/Caddyfile @@ -0,0 +1,7 @@ +localhost { + log { + output net 127.0.0.1:7500 + format json + } + respond "Hello there!" +} diff --git a/test/log.jsonl b/test/log.jsonl new file mode 100644 index 0000000..802beea --- /dev/null +++ b/test/log.jsonl @@ -0,0 +1,2 @@ +{"level":"info","ts":1680281540.1000268,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"47721","proto":"HTTP/3.0","method":"GET","host":"localhost","uri":"/?balls","headers":{"Alt-Used":["localhost"],"Cookie":[],"Priority":["u=1"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],"Sec-Fetch-Site":["none"],"Sec-Fetch-Mode":["navigate"],"Pragma":["no-cache"],"Accept-Language":["en-GB,en;q=0.5"],"Accept-Encoding":["gzip, deflate, br"],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-User":["?1"],"Cache-Control":["no-cache"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h3","server_name":"localhost"}},"user_id":"","duration":0.000019051,"size":12,"status":200,"resp_headers":{"Server":["Caddy"],"Content-Type":["text/plain; charset=utf-8"]}} +{"level":"info","ts":1680281540.3660147,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"47721","proto":"HTTP/3.0","method":"GET","host":"localhost","uri":"/favicon.ico","headers":{"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0"],"Accept-Language":["en-GB,en;q=0.5"],"Cookie":[],"Sec-Fetch-Dest":["image"],"Sec-Fetch-Mode":["no-cors"],"Cache-Control":["no-cache"],"Priority":["u=6"],"Accept":["image/avif,image/webp,*/*"],"Accept-Encoding":["gzip, deflate, br"],"Alt-Used":["localhost"],"Referer":["https://localhost/?balls"],"Sec-Fetch-Site":["same-origin"],"Pragma":["no-cache"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h3","server_name":"localhost"}},"user_id":"","duration":0.000061116,"size":12,"status":200,"resp_headers":{"Server":["Caddy"],"Content-Type":["text/plain; charset=utf-8"]}} \ No newline at end of file diff --git a/test/logs2.jsonl b/test/logs2.jsonl new file mode 100644 index 0000000..6df29f8 --- /dev/null +++ b/test/logs2.jsonl @@ -0,0 +1,2 @@ +{"level":"info","ts":1680281834.4265301,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"44882","proto":"HTTP/3.0","method":"GET","host":"localhost","uri":"/?balls","headers":{"Accept-Language":["en-GB,en;q=0.5"],"Alt-Used":["localhost"],"Cookie":[],"Upgrade-Insecure-Requests":["1"],"Sec-Fetch-Mode":["navigate"],"Pragma":["no-cache"],"Priority":["u=1"],"Accept":["text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"],"Accept-Encoding":["gzip, deflate, br"],"Sec-Fetch-Dest":["document"],"Sec-Fetch-Site":["cross-site"],"Cache-Control":["no-cache"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h3","server_name":"localhost"}},"user_id":"","duration":0.000022127,"size":12,"status":200,"resp_headers":{"Server":["Caddy"],"Content-Type":["text/plain; charset=utf-8"]}} +{"level":"info","ts":1680281834.5157068,"logger":"http.log.access.log0","msg":"handled request","request":{"remote_ip":"127.0.0.1","remote_port":"44882","proto":"HTTP/3.0","method":"GET","host":"localhost","uri":"/favicon.ico","headers":{"Referer":["https://localhost/?balls"],"Cookie":[],"Sec-Fetch-Dest":["image"],"Sec-Fetch-Site":["same-origin"],"Pragma":["no-cache"],"User-Agent":["Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/112.0"],"Accept":["image/avif,image/webp,*/*"],"Accept-Encoding":["gzip, deflate, br"],"Cache-Control":["no-cache"],"Sec-Fetch-Mode":["no-cors"],"Priority":["u=6"],"Accept-Language":["en-GB,en;q=0.5"],"Alt-Used":["localhost"]},"tls":{"resumed":false,"version":772,"cipher_suite":4865,"proto":"h3","server_name":"localhost"}},"user_id":"","duration":0.000020068,"size":12,"status":200,"resp_headers":{"Content-Type":["text/plain; charset=utf-8"],"Server":["Caddy"]}} diff --git a/test/sinkhole.py b/test/sinkhole.py new file mode 100644 index 0000000..ba49fa8 --- /dev/null +++ b/test/sinkhole.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import socket +import sys + +SOCKET=7502 + +serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +serversocket.bind(('localhost', SOCKET)) +serversocket.listen(5) # become a server socket, maximum 5 connections + +print(f"[*] Alive at localhost:{SOCKET}", file=sys.stderr) + +while True: + connection, address = serversocket.accept() + print(f"[+] New connection {address=}", file=sys.stderr) + while True: + buf = connection.recv(64) + if len(buf) > 0: + print(buf) + else: + connection.close() + print(f"[-] Connection closed", file=sys.stderr) + break