Remove reading list
This commit is contained in:
parent
b3829bccd7
commit
89e5c2a85f
5 changed files with 0 additions and 536 deletions
|
|
@ -18,11 +18,6 @@ func (h *HTTP) Address() string {
|
|||
return fmt.Sprintf("%s:%d", h.Host, h.Port)
|
||||
}
|
||||
|
||||
type ReadingList struct {
|
||||
Token string
|
||||
GitlabAccessToken string
|
||||
}
|
||||
|
||||
type SpotifyTiles struct {
|
||||
ClientID string
|
||||
ClientSecret string
|
||||
|
|
@ -42,7 +37,6 @@ type CertificateMonitor struct {
|
|||
type Config struct {
|
||||
Debug bool
|
||||
HTTP *HTTP
|
||||
ReadingList *ReadingList
|
||||
SpotifyTiles *SpotifyTiles
|
||||
CertificateMonitor *CertificateMonitor
|
||||
HostSuffix string
|
||||
|
|
@ -68,10 +62,6 @@ func Get() *Config {
|
|||
Host: cl.WithDefault("http.host", "127.0.0.1").AsString(),
|
||||
Port: cl.WithDefault("http.port", 8080).AsInt(),
|
||||
},
|
||||
ReadingList: &ReadingList{
|
||||
Token: cl.Required("readingList.token").AsString(),
|
||||
GitlabAccessToken: cl.Required("readingList.gitlabAccessToken").AsString(),
|
||||
},
|
||||
SpotifyTiles: &SpotifyTiles{
|
||||
ClientID: cl.Required("spotifyTiles.clientID").AsString(),
|
||||
ClientSecret: cl.Required("spotifyTiles.clientSecret").AsString(),
|
||||
|
|
|
|||
1
main.go
1
main.go
|
|
@ -15,7 +15,6 @@ import (
|
|||
|
||||
_ "github.com/codemicro/platform/modules/avatars"
|
||||
_ "github.com/codemicro/platform/modules/certificateMonitor"
|
||||
_ "github.com/codemicro/platform/modules/readingList"
|
||||
_ "github.com/codemicro/platform/modules/spotifyTiles"
|
||||
_ "github.com/codemicro/platform/modules/test"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,277 +0,0 @@
|
|||
package readingList
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
const readingListFilename = "readingList.csv"
|
||||
const mapFilename = "readingList.map.json"
|
||||
|
||||
func addRowToCSV(data *inputs) error {
|
||||
if err := data.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hnURL, err := queryHackerNews(data.URL)
|
||||
if err != nil {
|
||||
slog.Warn("Unable to search Hacker News", "url", data.URL)
|
||||
}
|
||||
|
||||
// if CSV file does not exist, create it with a header
|
||||
csvFilePath := store.MakePath(readingListFilename)
|
||||
|
||||
var doesCSVExist bool
|
||||
{
|
||||
_, err := os.Stat(csvFilePath)
|
||||
doesCSVExist = err == nil
|
||||
}
|
||||
|
||||
fileFlags := os.O_APPEND | os.O_WRONLY
|
||||
|
||||
var records [][]string
|
||||
|
||||
if !doesCSVExist {
|
||||
fileFlags = fileFlags | os.O_CREATE
|
||||
records = append(records, []string{"url", "title", "description", "image", "date", "hnurl"})
|
||||
}
|
||||
|
||||
timeStr, _ := time.Now().MarshalText()
|
||||
records = append(records, []string{data.URL, data.Title, data.Description, data.Image, string(timeStr), hnURL})
|
||||
|
||||
// make changes to CSV file
|
||||
|
||||
f, err := os.OpenFile(csvFilePath, fileFlags, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w := csv.NewWriter(f)
|
||||
_ = w.WriteAll(records)
|
||||
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var hnHTTPClient = new(http.Client)
|
||||
|
||||
type hackerNewsEntry struct {
|
||||
ObjectID string `json:"objectID"`
|
||||
Points int `json:"points"`
|
||||
}
|
||||
|
||||
var hackerNewsSubmissionURL = "https://news.ycombinator.com/item?id=%s"
|
||||
|
||||
func queryHackerNews(url string) (string, error) {
|
||||
|
||||
req, err := http.NewRequest("GET", "https://hn.algolia.com/api/v1/search", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// why does this fel so hacky
|
||||
queryParams := req.URL.Query()
|
||||
queryParams.Add("restrictSearchableAttributes", "url")
|
||||
queryParams.Add("hitsPerPage", "1000")
|
||||
queryParams.Add("query", url)
|
||||
req.URL.RawQuery = queryParams.Encode()
|
||||
|
||||
resp, err := hnHTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return "", fmt.Errorf("HN Search returned a non-200 status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
responseBody, _ := ioutil.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
|
||||
var x struct {
|
||||
Hits []*hackerNewsEntry `json:"hits"`
|
||||
}
|
||||
|
||||
err = json.Unmarshal(responseBody, &x)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var targetSubmission *hackerNewsEntry
|
||||
if len(x.Hits) == 0 {
|
||||
return "", nil
|
||||
} else if len(x.Hits) == 1 {
|
||||
targetSubmission = x.Hits[0]
|
||||
} else {
|
||||
// must be more than one hit
|
||||
var topRatedSubmission *hackerNewsEntry
|
||||
for _, entry := range x.Hits {
|
||||
if topRatedSubmission == nil || entry.Points > topRatedSubmission.Points {
|
||||
topRatedSubmission = entry
|
||||
}
|
||||
}
|
||||
targetSubmission = topRatedSubmission
|
||||
}
|
||||
|
||||
return fmt.Sprintf(hackerNewsSubmissionURL, targetSubmission.ObjectID), nil
|
||||
}
|
||||
|
||||
// openReadingListFile opens the reading list CSV for reading.
|
||||
func openReadingListFile() (*os.File, error) {
|
||||
csvFilePath := store.MakePath(readingListFilename)
|
||||
f, err := os.Open(csvFilePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func generateMapFile() error {
|
||||
f, err := openReadingListFile()
|
||||
if err != nil {
|
||||
return fmt.Errorf("open reading list CSV: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// This entire function presumes that the dates contained in the file are in a strictly increasing order the further
|
||||
// into the file you get.
|
||||
|
||||
type fileRange struct {
|
||||
Start int64 `json:"start"`
|
||||
End int64 `json:"end"`
|
||||
}
|
||||
|
||||
offsets := make(map[[2]int]*fileRange)
|
||||
|
||||
reader := csv.NewReader(f)
|
||||
reader.FieldsPerRecord = -1 // disable record length checking
|
||||
|
||||
_, _ = reader.Read() // ignore the header line
|
||||
|
||||
lineStart := reader.InputOffset()
|
||||
|
||||
for {
|
||||
var stop bool
|
||||
record, err := reader.Read()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
stop = true
|
||||
} else {
|
||||
return fmt.Errorf("read record: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(record) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
lineEnd := reader.InputOffset()
|
||||
|
||||
dateString := record[4]
|
||||
recordTime := &time.Time{}
|
||||
if err := recordTime.UnmarshalText([]byte(dateString)); err != nil {
|
||||
return fmt.Errorf("unmarshal time: %w", err)
|
||||
}
|
||||
|
||||
key := [2]int{int(recordTime.Month()), recordTime.Year()}
|
||||
|
||||
fr := offsets[key]
|
||||
if fr == nil {
|
||||
fr = &fileRange{}
|
||||
offsets[key] = fr
|
||||
fr.Start = lineStart
|
||||
}
|
||||
|
||||
fr.End = lineEnd - 1
|
||||
|
||||
if stop {
|
||||
break
|
||||
}
|
||||
|
||||
lineStart = lineEnd
|
||||
}
|
||||
|
||||
var months [][2]int
|
||||
for k := range offsets {
|
||||
months = append(months, k)
|
||||
}
|
||||
|
||||
sort.Slice(months, func(i, j int) bool {
|
||||
im, jm := months[i], months[j]
|
||||
if im[1] != jm[1] {
|
||||
return im[1] < jm[1]
|
||||
}
|
||||
return im[0] < jm[0]
|
||||
})
|
||||
|
||||
type resp struct {
|
||||
Name string `json:"name"`
|
||||
Range *fileRange `json:"range"`
|
||||
}
|
||||
|
||||
var res []*resp
|
||||
|
||||
for _, k := range months {
|
||||
res = append(res, &resp{
|
||||
Name: fmt.Sprintf("%s %d", time.Month(k[0]).String()[0:3], k[1]),
|
||||
Range: offsets[k],
|
||||
})
|
||||
}
|
||||
|
||||
j, err := json.Marshal(res)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal json: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(store.MakePath(mapFilename), j, 0644); err != nil {
|
||||
return fmt.Errorf("dump to file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func readLastNLines(f *os.File, n int) ([]byte, error) {
|
||||
st, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat reading list file: %w", err)
|
||||
}
|
||||
|
||||
var (
|
||||
buf []byte
|
||||
b = make([]byte, 1)
|
||||
newlineCount int
|
||||
fileSize = st.Size()
|
||||
)
|
||||
|
||||
for i := 0; int64(i) < fileSize; i += 1 {
|
||||
targetByte := fileSize - int64(i+1)
|
||||
_, err := f.ReadAt(b, targetByte)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read from reading list file: %w", err)
|
||||
}
|
||||
|
||||
if b[0] == '\n' {
|
||||
newlineCount += 1
|
||||
}
|
||||
|
||||
if newlineCount == n {
|
||||
break
|
||||
}
|
||||
|
||||
buf = append(b, buf...)
|
||||
}
|
||||
|
||||
return buf, nil
|
||||
}
|
||||
|
|
@ -1,210 +0,0 @@
|
|||
package readingList
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/carlmjohnson/requests"
|
||||
"github.com/codemicro/platform/config"
|
||||
"github.com/codemicro/platform/platform/util/htmlutil"
|
||||
"github.com/go-playground/validator"
|
||||
"github.com/julienschmidt/httprouter"
|
||||
g "github.com/maragudk/gomponents"
|
||||
"github.com/maragudk/gomponents/html"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func addHandler(rw http.ResponseWriter, rq *http.Request, _ httprouter.Params) error {
|
||||
//rw.Header().Set("Access-Control-Allow-Headers", "content-type,authorization")
|
||||
rw.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
rw.Header().Set("Content-Type", "text/html")
|
||||
|
||||
data := &struct {
|
||||
inputs
|
||||
NextURL string `validate:"required,url"`
|
||||
Token string `validate:"required"`
|
||||
}{
|
||||
inputs: inputs{
|
||||
URL: rq.URL.Query().Get("url"),
|
||||
Title: rq.URL.Query().Get("title"),
|
||||
Description: rq.URL.Query().Get("description"),
|
||||
Image: rq.URL.Query().Get("image"),
|
||||
},
|
||||
NextURL: rq.URL.Query().Get("nexturl"),
|
||||
Token: rq.URL.Query().Get("token"),
|
||||
}
|
||||
|
||||
{
|
||||
validate := validator.New()
|
||||
err := validate.Struct(data)
|
||||
if err != nil {
|
||||
rw.WriteHeader(400)
|
||||
n := htmlutil.BasePage("Bad request", g.Text("Bad request"), html.Br(), htmlutil.UnorderedList(strings.Split(err.Error(), "\n")))
|
||||
return n.Render(rw)
|
||||
}
|
||||
}
|
||||
|
||||
if subtle.ConstantTimeCompare([]byte(data.Token), []byte(config.Get().ReadingList.Token)) == 0 {
|
||||
rw.WriteHeader(401)
|
||||
n := htmlutil.BasePage("Invalid token", g.Text("Unauthorised - invalid token"))
|
||||
return n.Render(rw)
|
||||
}
|
||||
|
||||
serialisedInputs, err := json.Marshal(data.inputs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
type gitlabVar struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
bodyData := &struct {
|
||||
Ref string `json:"ref"`
|
||||
Variables []*gitlabVar `json:"variables"`
|
||||
}{
|
||||
Ref: "master",
|
||||
Variables: []*gitlabVar{
|
||||
{Key: "RL_INPUT_JSON", Value: string(serialisedInputs)},
|
||||
},
|
||||
}
|
||||
|
||||
rctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
err = requests.
|
||||
URL("https://git.tdpain.net/api/v4/projects/64/pipeline").
|
||||
Post().
|
||||
Header("Authorization", "Bearer "+config.Get().ReadingList.GitlabAccessToken).
|
||||
BodyJSON(bodyData).
|
||||
Fetch(rctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := addRowToCSV(&data.inputs); err != nil {
|
||||
return fmt.Errorf("add row to CSV: %w", err)
|
||||
}
|
||||
|
||||
if err := generateMapFile(); err != nil {
|
||||
return fmt.Errorf("generate map file: %w", err)
|
||||
}
|
||||
|
||||
return htmlutil.BasePage("Success!", html.Span(
|
||||
html.StyleAttr("color: darkgreen;"),
|
||||
g.Text("Success!"),
|
||||
),
|
||||
html.Script(g.Rawf(`setTimeout(function(){window.location.replace(%#v);}, 500);`, data.NextURL)),
|
||||
).Render(rw)
|
||||
}
|
||||
|
||||
func indexHandler(rw http.ResponseWriter, _ *http.Request, _ httprouter.Params) error {
|
||||
f, err := openReadingListFile()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
buf, err := readLastNLines(f, 6)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rd := csv.NewReader(bytes.NewReader(buf[:len(buf)-1]))
|
||||
records, err := rd.ReadAll()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
slices.Reverse(records)
|
||||
|
||||
ix := len(records)
|
||||
|
||||
return htmlutil.BasePage(
|
||||
"Reading list",
|
||||
html.H1(g.Text("Reading list")),
|
||||
html.A(g.Text("Source CSV"), g.Attr("href", "/csv")),
|
||||
html.H2(g.Text("Recent additions")),
|
||||
html.Ul(g.Map(records, func(i []string) g.Node {
|
||||
ix -= 1
|
||||
return html.Li(g.Text(i[1]+" "), html.A(g.Attr("href", fmt.Sprintf("/delete?n=%d", ix)), g.Text("[remove]")))
|
||||
})...),
|
||||
).Render(rw)
|
||||
}
|
||||
|
||||
func sourceCSVHandler(rw http.ResponseWriter, rq *http.Request, _ httprouter.Params) error {
|
||||
rw.Header().Set("Access-Control-Allow-Headers", "range")
|
||||
rw.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
|
||||
f, err := openReadingListFile()
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
rw.WriteHeader(404)
|
||||
_, _ = rw.Write([]byte("Not found"))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
rw.Header().Set("Content-Type", "text/csv")
|
||||
|
||||
rangeString := rq.Header.Get("Range")
|
||||
sp := strings.Split(rangeString, "=")
|
||||
if len(sp) == 2 && sp[0] == "bytes" {
|
||||
offsetStrings := strings.Split(sp[1], "-")
|
||||
start, err := strconv.ParseInt(offsetStrings[0], 10, 64)
|
||||
if err != nil {
|
||||
goto fullFile
|
||||
}
|
||||
_, err = f.Seek(start, 0)
|
||||
if err != nil {
|
||||
goto fullFile
|
||||
}
|
||||
if len(offsetStrings) > 1 {
|
||||
end, err := strconv.ParseInt(offsetStrings[1], 10, 64)
|
||||
if err != nil || end < start {
|
||||
_, _ = f.Seek(0, 0)
|
||||
goto fullFile
|
||||
}
|
||||
rw.WriteHeader(http.StatusPartialContent)
|
||||
_, err = io.CopyN(rw, f, end-start)
|
||||
return err
|
||||
}
|
||||
rw.WriteHeader(http.StatusPartialContent)
|
||||
}
|
||||
|
||||
fullFile:
|
||||
_, err = io.Copy(rw, f)
|
||||
return err
|
||||
}
|
||||
|
||||
func mapHandler(rw http.ResponseWriter, _ *http.Request, _ httprouter.Params) error {
|
||||
rw.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
|
||||
f, err := os.Open(store.MakePath(mapFilename))
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
rw.WriteHeader(404)
|
||||
_, _ = rw.Write([]byte("Not found"))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
rw.Header().Set("Content-Type", "application/json")
|
||||
|
||||
_, err = io.Copy(rw, f)
|
||||
return err
|
||||
}
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
package readingList
|
||||
|
||||
import (
|
||||
"github.com/codemicro/platform/platform"
|
||||
"github.com/codemicro/platform/platform/storage"
|
||||
"github.com/codemicro/platform/platform/util"
|
||||
"github.com/go-playground/validator"
|
||||
"github.com/julienschmidt/httprouter"
|
||||
)
|
||||
|
||||
const moduleName = "readinglist"
|
||||
|
||||
func init() {
|
||||
router.GET("/", util.WrapHandler(indexHandler))
|
||||
router.GET("/csv", util.WrapHandler(sourceCSVHandler))
|
||||
router.GET("/map", util.WrapHandler(mapHandler))
|
||||
router.GET("/api/add", util.WrapHandler(addHandler))
|
||||
|
||||
platform.RegisterProvider(moduleName, router, nil)
|
||||
}
|
||||
|
||||
var (
|
||||
router = httprouter.New()
|
||||
store = util.Must(storage.New(moduleName))
|
||||
)
|
||||
|
||||
var validate = validator.New()
|
||||
|
||||
type inputs struct {
|
||||
URL string `validate:"required,url"`
|
||||
Title string `validate:"required"`
|
||||
Description string
|
||||
Image string
|
||||
}
|
||||
|
||||
func (i *inputs) Validate() error {
|
||||
return validate.Struct(i)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue