mongoify and simplify
This commit is contained in:
parent
6d6a590211
commit
1f52bfb19a
8 changed files with 271 additions and 398 deletions
|
|
@ -1,54 +1,23 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"git.tdpain.net/codemicro/readingList/models"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"go.akpain.net/cfger"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Token string
|
||||
HTTPAddress string
|
||||
DatabaseFilename string
|
||||
PalmatumAuthentication string
|
||||
PalmatumSiteName string
|
||||
Token string
|
||||
HTTPAddress string
|
||||
MongoDSN string
|
||||
MongoDatabase string
|
||||
}
|
||||
|
||||
func Get() (*Config, error) {
|
||||
cl := cfger.New()
|
||||
var conf = &Config{
|
||||
Token: cl.GetEnv("READINGLISTD_INGEST_TOKEN").Required().AsString(),
|
||||
HTTPAddress: cl.GetEnv("READINGLISTD_HTTP_ADDR").WithDefault(":9231").AsString(),
|
||||
DatabaseFilename: cl.GetEnv("READINGLISTD_DATABASE_FILENAME").WithDefault("readinglist.sqlite3.db").AsString(),
|
||||
PalmatumAuthentication: cl.GetEnv("READINGLISTD_PALMATUM_AUTH").Required().AsString(),
|
||||
PalmatumSiteName: cl.GetEnv("READINGLISTD_SITE_NAME").Required().AsString(),
|
||||
Token: cl.GetEnv("READINGLISTD_INGEST_TOKEN").Required().AsString(),
|
||||
HTTPAddress: cl.GetEnv("READINGLISTD_HTTP_ADDR").WithDefault(":9231").AsString(),
|
||||
MongoDSN: cl.GetEnv("READINGLISTD_MONGO_DSN").WithDefault("mongodb://localhost/readinglist").AsString(),
|
||||
MongoDatabase: cl.GetEnv("READINGLISTD_MONGO_DATABASE").WithDefault("readinglist").AsString(),
|
||||
}
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
type ArticleChannelWrapper struct {
|
||||
Article *models.NewArticle
|
||||
finishedChannel chan error
|
||||
}
|
||||
|
||||
func NewArticleChannelWrapper(a *models.NewArticle) *ArticleChannelWrapper {
|
||||
return &ArticleChannelWrapper{
|
||||
Article: a,
|
||||
finishedChannel: make(chan error, 1),
|
||||
}
|
||||
}
|
||||
|
||||
func (a *ArticleChannelWrapper) Finish(e error) {
|
||||
a.finishedChannel <- e
|
||||
close(a.finishedChannel)
|
||||
}
|
||||
|
||||
func (a *ArticleChannelWrapper) Error() error {
|
||||
return <-a.finishedChannel
|
||||
}
|
||||
|
||||
type ModuleContext struct {
|
||||
DB *sqlx.DB
|
||||
Config *Config
|
||||
NewArticleChannel chan *ArticleChannelWrapper
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,96 +1,80 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"context"
|
||||
"fmt"
|
||||
"git.tdpain.net/codemicro/readingList/cmd/readinglistd/internal/config"
|
||||
"git.tdpain.net/codemicro/readingList/models"
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||
"time"
|
||||
)
|
||||
|
||||
const programSchemaVersion = 2
|
||||
type DB struct {
|
||||
client *mongo.Client
|
||||
database *mongo.Database
|
||||
}
|
||||
|
||||
func NewDB(fname string) (*sqlx.DB, error) {
|
||||
db, err := sqlx.Connect("sqlite3", fname)
|
||||
func NewDB(conf *config.Config) (*DB, error) {
|
||||
client, err := mongo.Connect(options.Client().ApplyURI(conf.MongoDSN))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open database: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS schema_version(
|
||||
"n" integer not null
|
||||
)`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create schema_version table: %w", err)
|
||||
}
|
||||
|
||||
var currentSchemaVersion int
|
||||
if err := db.QueryRowx("SELECT n FROM schema_version").Scan(¤tSchemaVersion); err != nil {
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return nil, fmt.Errorf("unable to read schema version from database: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
switch currentSchemaVersion {
|
||||
case 0:
|
||||
// Note that version 0 did not originally include a schema_version mechanism so a v0 db so this statement must be if not exists
|
||||
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS articles(
|
||||
"id" varchar not null primary key,
|
||||
"date" datetime not null,
|
||||
"url" varchar not null,
|
||||
"title" varchar not null,
|
||||
"description" varchar,
|
||||
"image_url" varchar,
|
||||
"hacker_news_url" varchar
|
||||
)`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create articles table: %w", err)
|
||||
}
|
||||
currentSchemaVersion = 1
|
||||
fallthrough
|
||||
case 1:
|
||||
_, err = db.Exec(`ALTER TABLE articles ADD COLUMN is_favourite INTEGER NOT NULL DEFAULT FALSE`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("add is_favourite to articles table: %w", err)
|
||||
}
|
||||
currentSchemaVersion = 2
|
||||
fallthrough
|
||||
case programSchemaVersion:
|
||||
// noop
|
||||
}
|
||||
|
||||
_, err = db.Exec(`DELETE FROM schema_version`)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("delete old schema version number: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.Exec(`INSERT INTO schema_version(n) VALUES (?)`, programSchemaVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("insert schema version number: %w", err)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
return &DB{client: client, database: client.Database(conf.MongoDatabase)}, nil
|
||||
}
|
||||
|
||||
func InsertArticle(db *sqlx.DB, article *models.Article) error {
|
||||
_, err := db.NamedExec(
|
||||
`INSERT INTO articles("id", "date", "url", "title", "description", "image_url", "hacker_news_url", "is_favourite") VALUES (:id, :date, :url, :title, :description, :image_url, :hacker_news_url, :is_favourite)`,
|
||||
article,
|
||||
)
|
||||
func (db *DB) InsertArticle(article *models.Article) error {
|
||||
res, err := db.database.Collection("articles").InsertOne(context.Background(), article)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert article: %w", err)
|
||||
return fmt.Errorf("insert new article: %w", err)
|
||||
}
|
||||
fmt.Printf("ID: %v\n", res.InsertedID)
|
||||
article.ID = res.InsertedID
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetAllArticles(db *sqlx.DB) ([]*models.Article, error) {
|
||||
articles := []*models.Article{}
|
||||
err := db.Select(&articles, `SELECT * FROM articles`)
|
||||
func (db *DB) GetAllArticles() ([]*models.Article, error) {
|
||||
cursor, err := db.database.Collection("articles").Aggregate(context.Background(), mongo.Pipeline{
|
||||
bson.D{{"$sort", bson.D{{"date", 1}}}},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("select all articles: %w", err)
|
||||
return nil, fmt.Errorf("find all articles: %w", err)
|
||||
}
|
||||
// if err := res.StructScan(&articles); err != nil {
|
||||
// return nil, fmt.Errorf("scan article results: %w", err)
|
||||
// }
|
||||
return articles, nil
|
||||
defer cursor.Close(context.Background())
|
||||
|
||||
var results []*models.Article
|
||||
if err := cursor.All(context.Background(), &results); err != nil {
|
||||
return nil, fmt.Errorf("read all articles: %w", err)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetArticlesForMonth(year int, month int) ([]*models.Article, error) {
|
||||
start := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.UTC)
|
||||
var end time.Time
|
||||
if month+1 > 12 {
|
||||
end = time.Date(year+1, time.Month(1), 1, 0, 0, 0, 0, time.UTC)
|
||||
} else {
|
||||
end = time.Date(year, time.Month(month+1), 1, 0, 0, 0, 0, time.UTC)
|
||||
}
|
||||
|
||||
cursor, err := db.database.Collection("articles").Aggregate(context.Background(), mongo.Pipeline{
|
||||
bson.D{{"$match", bson.D{{"date", bson.D{{"$gt", start}}}}}},
|
||||
bson.D{{"$match", bson.D{{"date", bson.D{{"$lt", end}}}}}},
|
||||
bson.D{{"$sort", bson.D{{"date", 1}}}},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("find all articles: %w", err)
|
||||
}
|
||||
defer cursor.Close(context.Background())
|
||||
|
||||
var results []*models.Article
|
||||
if err := cursor.All(context.Background(), &results); err != nil {
|
||||
return nil, fmt.Errorf("read all articles: %w", err)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,8 +3,10 @@ package http
|
|||
import (
|
||||
"crypto/subtle"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.tdpain.net/codemicro/readingList/cmd/readinglistd/internal/config"
|
||||
"git.tdpain.net/codemicro/readingList/cmd/readinglistd/internal/database"
|
||||
"git.tdpain.net/codemicro/readingList/models"
|
||||
"github.com/go-playground/validator"
|
||||
g "github.com/maragudk/gomponents"
|
||||
|
|
@ -12,14 +14,16 @@ import (
|
|||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Listen(mctx *config.ModuleContext) error {
|
||||
slog.Info("starting HTTP server", "address", mctx.Config.HTTPAddress)
|
||||
func Listen(conf *config.Config, db *database.DB) error {
|
||||
slog.Info("starting HTTP server", "address", conf.HTTPAddress)
|
||||
|
||||
e := &endpoints{mctx}
|
||||
e := &endpoints{DB: db, Config: conf}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
|
|
@ -37,18 +41,19 @@ func Listen(mctx *config.ModuleContext) error {
|
|||
}
|
||||
}))
|
||||
|
||||
//mux.Handle("POST /generate", http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
|
||||
// if err := e.generate(rw, req); err != nil {
|
||||
// slog.Error("error in generate HTTP handler", "error", err, "request", req)
|
||||
// rw.WriteHeader(http.StatusInternalServerError)
|
||||
// }
|
||||
//}))
|
||||
mux.Handle("GET /list", http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
|
||||
if err := e.list(rw, req); err != nil {
|
||||
slog.Error("error in list HTTP handler", "error", err, "request", req)
|
||||
rw.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
}))
|
||||
|
||||
return http.ListenAndServe(mctx.Config.HTTPAddress, mux)
|
||||
return http.ListenAndServe(conf.HTTPAddress, mux)
|
||||
}
|
||||
|
||||
type endpoints struct {
|
||||
*config.ModuleContext
|
||||
DB *database.DB
|
||||
Config *config.Config
|
||||
}
|
||||
|
||||
// directIngest is an ingest endpoint that accepts JSON-encoded bodies.
|
||||
|
|
@ -81,10 +86,7 @@ func (e endpoints) directIngest(rw http.ResponseWriter, req *http.Request) error
|
|||
return nil
|
||||
}
|
||||
|
||||
job := config.NewArticleChannelWrapper(requestData)
|
||||
e.NewArticleChannel <- job
|
||||
|
||||
if err := job.Error(); err != nil {
|
||||
if err := processNewArticle(e.DB, requestData); err != nil {
|
||||
_, _ = rw.Write([]byte(err.Error()))
|
||||
rw.WriteHeader(http.StatusBadRequest)
|
||||
} else {
|
||||
|
|
@ -131,15 +133,11 @@ func (e endpoints) browserIngest(rw http.ResponseWriter, req *http.Request) erro
|
|||
return n.Render(rw)
|
||||
}
|
||||
|
||||
job := config.NewArticleChannelWrapper(&data.NewArticle)
|
||||
e.NewArticleChannel <- job
|
||||
|
||||
var page g.Node
|
||||
|
||||
if err := job.Error(); err != nil {
|
||||
if err := processNewArticle(e.DB, &data.NewArticle); err != nil {
|
||||
page = basePageWithBackgroundColour("Addition failed", "#fadbd8", P(
|
||||
StyleAttr("font-weight: bold;"),
|
||||
g.Text("Error: " + err.Error()),
|
||||
g.Text("Error: "+err.Error()),
|
||||
))
|
||||
} else {
|
||||
page = basePageWithBackgroundColour("Success!", "#d4efdf", P(
|
||||
|
|
@ -156,10 +154,39 @@ func (e endpoints) browserIngest(rw http.ResponseWriter, req *http.Request) erro
|
|||
Script(g.Raw(`setTimeout(function(){history.back();}, 750);`)),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
return page.Render(rw)
|
||||
}
|
||||
|
||||
func (e endpoints) list(rw http.ResponseWriter, req *http.Request) error {
|
||||
year, _ := strconv.Atoi(req.URL.Query().Get("year"))
|
||||
month, _ := strconv.Atoi(req.URL.Query().Get("month"))
|
||||
|
||||
var articles []*models.Article
|
||||
var err error
|
||||
|
||||
if year != 0 && month != 0 {
|
||||
articles, err = e.DB.GetArticlesForMonth(year, month)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get articles for month %d-%d: %w", year, month, err)
|
||||
}
|
||||
} else {
|
||||
articles, err = e.DB.GetAllArticles()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get all articles: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
res, err := json.Marshal(articles)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal all articles: %w", err)
|
||||
}
|
||||
|
||||
rw.Header().Set("Content-Type", "application/json")
|
||||
_, _ = rw.Write(res)
|
||||
return nil
|
||||
}
|
||||
|
||||
func basePageWithBackgroundColour(title, colour string, content ...g.Node) g.Node {
|
||||
styles := `body {
|
||||
font-family: sans-serif;
|
||||
|
|
@ -170,7 +197,7 @@ func basePageWithBackgroundColour(title, colour string, content ...g.Node) g.Nod
|
|||
styles += "background-color: " + colour + ";\n"
|
||||
}
|
||||
styles += "}"
|
||||
|
||||
|
||||
return HTML(
|
||||
Head(
|
||||
Meta(g.Attr("name", "viewport"), g.Attr("content", "width=device-width, initial-scale=1")),
|
||||
|
|
@ -191,6 +218,38 @@ func unorderedList(x []string) g.Node {
|
|||
})...)
|
||||
}
|
||||
|
||||
func processNewArticle(db *database.DB, newArticle *models.NewArticle) error {
|
||||
article := &models.Article{
|
||||
NewArticle: *newArticle,
|
||||
}
|
||||
|
||||
{ // remove fragment
|
||||
parsed, err := url.Parse(article.URL)
|
||||
if err != nil {
|
||||
return errors.New("invalid URL")
|
||||
}
|
||||
parsed.Fragment = ""
|
||||
article.URL = parsed.String()
|
||||
}
|
||||
|
||||
hnURL, err := queryHackerNews(article.URL)
|
||||
if err != nil {
|
||||
slog.Warn("unable to query hacker news", "error", err, "article", article)
|
||||
}
|
||||
article.HackerNewsURL = hnURL
|
||||
|
||||
if len(article.Description) > 500 {
|
||||
article.Description = article.Description[:500] + " [trimmed]"
|
||||
}
|
||||
|
||||
if err := db.InsertArticle(article); err != nil {
|
||||
slog.Error("unable to insert article", "error", err, "article", newArticle)
|
||||
return errors.New("fatal database error")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//func (e endpoints) generate(rw http.ResponseWriter, _ *http.Request) error {
|
||||
// if err := worker.GenerateSiteAndUpload(e.DB, e.Config); err != nil {
|
||||
// return err
|
||||
|
|
@ -198,3 +257,61 @@ func unorderedList(x []string) g.Node {
|
|||
// rw.WriteHeader(204)
|
||||
// return nil
|
||||
//}
|
||||
|
||||
// queryHackerNews searches the Hacker News index to find a submission with a matching URL to that provided.
|
||||
// If a submission is found, its URL is returned. If no submission is found, an empty string is returned. If multiple submissions are found, the URL of the one with the most points is returned.
|
||||
func queryHackerNews(queryURL string) (string, error) {
|
||||
queryParams := make(url.Values)
|
||||
queryParams.Add("restrictSearchableAttributes", "url")
|
||||
queryParams.Add("hitsPerPage", "1000")
|
||||
queryParams.Add("query", queryURL)
|
||||
|
||||
req, err := http.NewRequest("GET", "https://hn.algolia.com/api/v1/search?"+queryParams.Encode(), nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
resp, err := new(http.Client).Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("HN Search returned a non-200 status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
responseBody, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
|
||||
type hackerNewsEntry struct {
|
||||
ObjectID string `json:"objectID"`
|
||||
Points int `json:"points"`
|
||||
}
|
||||
|
||||
var x struct {
|
||||
Hits []*hackerNewsEntry `json:"hits"`
|
||||
}
|
||||
|
||||
err = json.Unmarshal(responseBody, &x)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var targetSubmission *hackerNewsEntry
|
||||
if len(x.Hits) == 0 {
|
||||
return "", nil
|
||||
} else if len(x.Hits) == 1 {
|
||||
targetSubmission = x.Hits[0]
|
||||
} else {
|
||||
// must be more than one hit
|
||||
var topRatedSubmission *hackerNewsEntry
|
||||
for _, entry := range x.Hits {
|
||||
if topRatedSubmission == nil || entry.Points > topRatedSubmission.Points {
|
||||
topRatedSubmission = entry
|
||||
}
|
||||
}
|
||||
targetSubmission = topRatedSubmission
|
||||
}
|
||||
|
||||
return fmt.Sprintf("https://news.ycombinator.com/item?id=%s", targetSubmission.ObjectID), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,241 +0,0 @@
|
|||
package worker
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.tdpain.net/codemicro/readingList/cmd/readinglistd/internal/config"
|
||||
"git.tdpain.net/codemicro/readingList/cmd/readinglistd/internal/database"
|
||||
"git.tdpain.net/codemicro/readingList/cmd/readinglistd/internal/generator"
|
||||
"io"
|
||||
"log/slog"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
"errors"
|
||||
"git.tdpain.net/codemicro/readingList/models"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jmoiron/sqlx"
|
||||
)
|
||||
|
||||
func RunSiteWorker(mctx *config.ModuleContext) {
|
||||
go siteWorker(mctx)
|
||||
}
|
||||
|
||||
func siteWorker(mctx *config.ModuleContext) {
|
||||
var job *config.ArticleChannelWrapper
|
||||
rootLoop:
|
||||
for {
|
||||
job = <-mctx.NewArticleChannel
|
||||
loop:
|
||||
for {
|
||||
article := &models.Article{
|
||||
NewArticle: *job.Article,
|
||||
ID: uuid.New(),
|
||||
}
|
||||
|
||||
{ // remove fragment
|
||||
parsed, err := url.Parse(article.URL)
|
||||
if err != nil {
|
||||
job.Finish(errors.New("invalid URL"))
|
||||
slog.Error("invalud URL supplied to worker", "url", article.URL)
|
||||
continue rootLoop
|
||||
}
|
||||
parsed.Fragment = ""
|
||||
article.URL = parsed.String()
|
||||
}
|
||||
|
||||
hnURL, err := queryHackerNews(article.URL)
|
||||
if err != nil {
|
||||
slog.Warn("unable to query hacker news", "error", err, "article", article)
|
||||
}
|
||||
article.HackerNewsURL = hnURL
|
||||
|
||||
if len(article.Description) > 500 {
|
||||
article.Description = article.Description[:500] + " [trimmed]"
|
||||
}
|
||||
|
||||
if err := database.InsertArticle(mctx.DB, article); err != nil {
|
||||
job.Finish(errors.New("fatal database error"))
|
||||
slog.Error("unable to insert article", "error", err, "article", article)
|
||||
continue rootLoop
|
||||
}
|
||||
|
||||
job.Finish(nil)
|
||||
|
||||
// The purpose of this is to delay rebuilding the site if another article appears in the next 20 seconds
|
||||
ticker := time.NewTicker(time.Second * 20)
|
||||
select {
|
||||
case <-ticker.C:
|
||||
ticker.Stop()
|
||||
break loop
|
||||
case job = <-mctx.NewArticleChannel:
|
||||
ticker.Stop()
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err := GenerateSiteAndUpload(mctx.DB, mctx.Config); err != nil {
|
||||
slog.Error("error while executing site generation", "error", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// queryHackerNews searches the Hacker News index to find a submission with a matching URL to that provided.
|
||||
// If a submission is found, its URL is returned. If no submission is found, an empty string is returned. If multiple submissions are found, the URL of the one with the most points is returned.
|
||||
func queryHackerNews(queryURL string) (string, error) {
|
||||
queryParams := make(url.Values)
|
||||
queryParams.Add("restrictSearchableAttributes", "url")
|
||||
queryParams.Add("hitsPerPage", "1000")
|
||||
queryParams.Add("query", queryURL)
|
||||
|
||||
req, err := http.NewRequest("GET", "https://hn.algolia.com/api/v1/search?"+queryParams.Encode(), nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
resp, err := new(http.Client).Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("HN Search returned a non-200 status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
responseBody, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
|
||||
type hackerNewsEntry struct {
|
||||
ObjectID string `json:"objectID"`
|
||||
Points int `json:"points"`
|
||||
}
|
||||
|
||||
var x struct {
|
||||
Hits []*hackerNewsEntry `json:"hits"`
|
||||
}
|
||||
|
||||
err = json.Unmarshal(responseBody, &x)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var targetSubmission *hackerNewsEntry
|
||||
if len(x.Hits) == 0 {
|
||||
return "", nil
|
||||
} else if len(x.Hits) == 1 {
|
||||
targetSubmission = x.Hits[0]
|
||||
} else {
|
||||
// must be more than one hit
|
||||
var topRatedSubmission *hackerNewsEntry
|
||||
for _, entry := range x.Hits {
|
||||
if topRatedSubmission == nil || entry.Points > topRatedSubmission.Points {
|
||||
topRatedSubmission = entry
|
||||
}
|
||||
}
|
||||
targetSubmission = topRatedSubmission
|
||||
}
|
||||
|
||||
return fmt.Sprintf("https://news.ycombinator.com/item?id=%s", targetSubmission.ObjectID), nil
|
||||
}
|
||||
|
||||
var siteGenerationLock sync.Mutex
|
||||
|
||||
func GenerateSiteAndUpload(db *sqlx.DB, conf *config.Config) error {
|
||||
siteGenerationLock.Lock()
|
||||
defer siteGenerationLock.Unlock()
|
||||
|
||||
allArticles, err := database.GetAllArticles(db)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to fetch all articles: %w", err)
|
||||
}
|
||||
|
||||
sitePath, err := generator.GenerateSite(allArticles)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to generate site: %w", err)
|
||||
}
|
||||
|
||||
siteZipFile, err := packageSite(sitePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to package site: %w", err)
|
||||
}
|
||||
|
||||
_ = os.RemoveAll(sitePath)
|
||||
|
||||
if err := uploadSite(conf, siteZipFile); err != nil {
|
||||
return fmt.Errorf("unable to upload site to palmatum: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func packageSite(sitePath string) (*bytes.Buffer, error) {
|
||||
dfs := os.DirFS(sitePath)
|
||||
buffer := new(bytes.Buffer)
|
||||
|
||||
writer := zip.NewWriter(buffer)
|
||||
if err := writer.AddFS(dfs); err != nil {
|
||||
return nil, fmt.Errorf("add fs to zip file: %w", err)
|
||||
}
|
||||
if err := writer.Flush(); err != nil {
|
||||
return nil, fmt.Errorf("flush zip writer: %w", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, fmt.Errorf("close zip writer: %w", err)
|
||||
}
|
||||
|
||||
return buffer, nil
|
||||
}
|
||||
|
||||
func uploadSite(conf *config.Config, reader io.Reader) error {
|
||||
bodyBuffer := new(bytes.Buffer)
|
||||
mpWriter := multipart.NewWriter(bodyBuffer)
|
||||
|
||||
if err := mpWriter.WriteField("slug", conf.PalmatumSiteName); err != nil {
|
||||
return fmt.Errorf("write field to multipart: %w", err)
|
||||
}
|
||||
|
||||
fieldWriter, err := mpWriter.CreateFormFile("bundle", "site.zip")
|
||||
if err != nil {
|
||||
return fmt.Errorf("create multipart field: %w", err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(fieldWriter, reader); err != nil {
|
||||
return fmt.Errorf("copy site file to multipart writer: %w", err)
|
||||
}
|
||||
|
||||
if err := mpWriter.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close multipart writer: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, "https://management.pages.tdpain.net/api/site/bundle", bodyBuffer)
|
||||
if err != nil {
|
||||
return fmt.Errorf("make http request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", mpWriter.FormDataContentType())
|
||||
req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(conf.PalmatumAuthentication)))
|
||||
|
||||
resp, err := (&http.Client{
|
||||
Timeout: time.Second * 10,
|
||||
}).Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("do http request: %w", err)
|
||||
}
|
||||
|
||||
bodyCont, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
|
||||
if category := resp.StatusCode / 100; !(category == 2 || category == 3) {
|
||||
slog.Info("upload error encountered", "body", string(bodyCont))
|
||||
return fmt.Errorf("got %d status code returned from Palmatum", resp.StatusCode)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -4,7 +4,6 @@ import (
|
|||
"git.tdpain.net/codemicro/readingList/cmd/readinglistd/internal/config"
|
||||
"git.tdpain.net/codemicro/readingList/cmd/readinglistd/internal/database"
|
||||
"git.tdpain.net/codemicro/readingList/cmd/readinglistd/internal/http"
|
||||
"git.tdpain.net/codemicro/readingList/cmd/readinglistd/internal/worker"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
|
|
@ -20,17 +19,10 @@ func run() error {
|
|||
return err
|
||||
}
|
||||
|
||||
db, err := database.NewDB(conf.DatabaseFilename)
|
||||
db, err := database.NewDB(conf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mctx := &config.ModuleContext{
|
||||
DB: db,
|
||||
Config: conf,
|
||||
NewArticleChannel: make(chan *config.ArticleChannelWrapper, 5),
|
||||
}
|
||||
|
||||
worker.RunSiteWorker(mctx)
|
||||
return http.Listen(mctx)
|
||||
|
||||
return http.Listen(conf, db)
|
||||
}
|
||||
|
|
|
|||
10
go.mod
10
go.mod
|
|
@ -15,7 +15,17 @@ require (
|
|||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/go-playground/locales v0.14.0 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.0 // indirect
|
||||
github.com/golang/snappy v1.0.0 // indirect
|
||||
github.com/klauspost/compress v1.16.7 // indirect
|
||||
github.com/leodido/go-urn v1.2.1 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.mongodb.org/mongo-driver/v2 v2.2.0 // indirect
|
||||
golang.org/x/crypto v0.33.0 // indirect
|
||||
golang.org/x/sync v0.11.0 // indirect
|
||||
golang.org/x/text v0.22.0 // indirect
|
||||
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
|
|
|||
43
go.sum
43
go.sum
|
|
@ -11,10 +11,14 @@ github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp
|
|||
github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig=
|
||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
|
||||
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
|
||||
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
|
||||
github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I=
|
||||
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
|
||||
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
|
|
@ -28,10 +32,49 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
|||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.akpain.net/cfger v0.2.0 h1:96/Uij1vWNCS3/7PEARLm4e+F5ictlQLzHsYHxl4MoI=
|
||||
go.akpain.net/cfger v0.2.0/go.mod h1:uaeo30IdnyNNBIEAT0SwvGIWGBauxMI+THVbk8L0oTs=
|
||||
go.mongodb.org/mongo-driver/v2 v2.2.0 h1:WwhNgGrijwU56ps9RtIsgKfGLEZeypxqbEYfThrBScM=
|
||||
go.mongodb.org/mongo-driver/v2 v2.2.0/go.mod h1:qQkDMhCGWl3FN509DfdPd4GRBLU/41zqF/k8eTRceps=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
|
||||
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
|
||||
golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
|
||||
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
|
||||
|
|
|
|||
|
|
@ -1,21 +1,20 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"github.com/google/uuid"
|
||||
"time"
|
||||
)
|
||||
|
||||
type NewArticle struct {
|
||||
URL string `validate:"required,url"`
|
||||
Title string `validate:"required"`
|
||||
Description string `db:"description"`
|
||||
ImageURL string `db:"image_url"`
|
||||
Date time.Time `db:"date" validate:"required"`
|
||||
IsFavourite bool `db:"is_favourite"`
|
||||
URL string `validate:"required,url"`
|
||||
Title string `validate:"required"`
|
||||
Description string
|
||||
ImageURL string `bson:"image_url"`
|
||||
Date time.Time `validate:"required"`
|
||||
IsFavourite bool `bson:"is_favourite"`
|
||||
}
|
||||
|
||||
type Article struct {
|
||||
NewArticle
|
||||
ID uuid.UUID
|
||||
HackerNewsURL string `db:"hacker_news_url"`
|
||||
NewArticle `bson:",inline"`
|
||||
ID any `bson:"_id,omitempty"`
|
||||
HackerNewsURL string `bson:"hacker_news_url"`
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue