initial commit: base structure, config, models

.gitignore

@@ -0,0 +1,28 @@
+
+# ---> Go
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# ---> VisualStudioCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/

.vscode/launch.json

@@ -0,0 +1,13 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [{
+        "name": "Launch Package",
+        "type": "go",
+        "request": "launch",
+        "mode": "auto",
+        "program": "${workspaceFolder}/cmd/feedizer"
+    }]
+}

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "cSpell.words": [
+        "adrg",
+        "feedizer"
+    ]
+}

README.md

@@ -0,0 +1,3 @@
+# Feedizer
+
+A rewrite of the [Feedizer](https://git.zom.bi/fanir/feedizer) project originally written in PHP5.

cmd/feedizer/config/config.go

@@ -0,0 +1,56 @@
+package config
+
+import (
+	"log"
+	"path/filepath"
+
+	"github.com/adrg/xdg"
+)
+
+type DatabaseConfig struct {
+	Socket   string
+	Host     string
+	Port     int
+	Username string
+	Password string
+	Database string
+	Options  string
+}
+
+type Config struct {
+	Database DatabaseConfig
+}
+
+var config = Config{
+	Database: DatabaseConfig{
+		Host:     "localhost",
+		Port:     5432,
+		Database: "feedizer",
+		Options:  "sslmode=require",
+	},
+}
+
+const appName = "feedizer"
+
+func Load() (*Config, error) {
+	numXdgDirs := len(xdg.ConfigDirs)
+	paths := make([]string, numXdgDirs+2)
+	for i, path := range xdg.ConfigDirs {
+		paths[numXdgDirs-i-1] = filepath.Join(path, appName, appName+".yaml")
+	}
+	paths[numXdgDirs] = filepath.Join(xdg.ConfigHome, appName, appName+".yaml")
+	paths[numXdgDirs+1] = appName + ".yaml"
+
+	used, err := ReadFiles(paths, &config)
+	if err != nil {
+		return nil, err
+	}
+	log.Println("config: search paths are:", paths)
+	log.Println("config: using config files:", used)
+
+	envAvailable, envUsed := ReadEnv(&config, appName)
+	log.Println("config: usable env vars:", envAvailable)
+	log.Println("config: using env vars:", envUsed)
+
+	return &config, nil
+}

cmd/feedizer/config/env.go

@@ -0,0 +1,68 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+func ReadEnv(c *Config, prefix string) (namesAvailable, namesUsed []string) {
+	return structFromEnv(reflect.ValueOf(c), reflect.TypeOf(c), prefix)
+}
+
+func structFromEnv(v reflect.Value, t reflect.Type, prefix string) (namesAvailable, namesUsed []string) {
+
+	if v.Kind() == reflect.Ptr {
+		v = v.Elem()
+		t = t.Elem()
+	}
+
+	for i := 0; i < v.NumField(); i++ {
+		fv := v.Field(i)
+		ft := t.Field(i)
+		tag := ft.Tag.Get("env")
+		if !ft.IsExported() || tag == "-" {
+			continue
+		}
+
+		name := ft.Name
+		if tag != "" {
+			name = tag
+		}
+
+		if fv.Kind() == reflect.Struct {
+			na, nu := structFromEnv(fv, ft.Type, prefix+"_"+name)
+			namesAvailable = append(namesAvailable, na...)
+			namesUsed = append(namesUsed, nu...)
+		} else if !ft.Anonymous {
+			envName := strings.ToUpper(prefix + "_" + name)
+			namesAvailable = append(namesAvailable, envName)
+
+			val, ok := os.LookupEnv(envName)
+			if !ok {
+				continue
+			}
+			if !fv.CanSet() {
+				panic("cannot set field for environment variable " + envName)
+			}
+
+			switch fk := fv.Kind(); fk {
+			case reflect.String:
+				fv.SetString(val)
+			case reflect.Int:
+				x, err := strconv.ParseInt(val, 10, 0)
+				if err != nil {
+					panic(err)
+				}
+				fv.SetInt(x)
+			default:
+				panic(fmt.Sprintf("cannot set field for environment variable %s: type %s is not supported", envName, fk))
+			}
+			namesUsed = append(namesUsed, envName)
+		}
+
+	}
+	return namesAvailable, namesUsed
+}

cmd/feedizer/config/files.go

@@ -0,0 +1,70 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/adrg/xdg"
+	"gopkg.in/yaml.v3"
+)
+
+var ErrUnsupportedFormat = errors.New("unsupported format")
+var ErrFileExists = errors.New("file already exists")
+
+func ReadFiles(files []string, out *Config) (usedFiles []string, err error) {
+	for _, path := range files {
+		f, err := os.Open(path)
+		if errors.Is(err, os.ErrNotExist) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("cannot read config file %s: %w", path, err)
+		}
+
+		ext := filepath.Ext(path)
+		if len(ext) > 0 {
+			ext = ext[1:]
+		}
+		switch ext {
+		case "yaml", "yml":
+			if err = yaml.NewDecoder(f).Decode(out); err != nil {
+				return nil, fmt.Errorf("cannot parse yaml config file %s: %w", path, err)
+			}
+		default:
+			return nil, fmt.Errorf("cannot read config file %s: %w", path, ErrUnsupportedFormat)
+		}
+
+		usedFiles = append(usedFiles, path)
+	}
+	return usedFiles, nil
+}
+
+func WriteFile(path string) (string, error) {
+	if path == "" {
+		var err error
+		if path, err = xdg.ConfigFile(filepath.Join(appName, appName+".yaml")); err != nil {
+			return "", err
+		}
+	}
+	println(path)
+
+	if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
+		if err == nil {
+			return "", ErrFileExists
+		}
+		return "", err
+	}
+
+	f, err := os.OpenFile(path, os.O_CREATE, 0o600)
+	if err != nil {
+		return "", err
+	}
+
+	if err = yaml.NewEncoder(f).Encode(config); err != nil {
+		return "", err
+	}
+
+	return path, f.Close()
+}

cmd/feedizer/database/database.go

@@ -0,0 +1,69 @@
+package database
+
+import (
+	"database/sql"
+	"fmt"
+
+	"git.zom.bi/fanir/feedizer/cmd/feedizer/config"
+	"git.zom.bi/fanir/feedizer/models/migrations"
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database/postgres"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+)
+
+var LogName string
+
+func Open(c config.DatabaseConfig) (*sql.DB, error) {
+	var dsn string
+	if c.Socket != "" {
+		dsn = fmt.Sprintf("postgres://%s:%s@%s/%s?%s", c.Username, c.Password, c.Socket, c.Database, c.Options)
+		LogName = fmt.Sprintf("%s/%s", c.Socket, c.Database)
+	} else {
+		dsn = fmt.Sprintf("postgres://%s:%s@%s:%d/%s?%s", c.Username, c.Password, c.Host, c.Port, c.Database, c.Options)
+		LogName = fmt.Sprintf("%s:%d/%s", c.Host, c.Port, c.Database)
+	}
+	return sql.Open("postgres", dsn)
+}
+
+func Migrate(db *sql.DB) error {
+	source, err := iofs.New(migrations.FS, ".")
+	if err != nil {
+		return err
+	}
+
+	driver, err := postgres.WithInstance(db, &postgres.Config{})
+	if err != nil {
+		return err
+	}
+
+	m, err := migrate.NewWithInstance("iofs", source, "postgres", driver)
+	if err != nil {
+		return err
+	}
+
+	return m.Up()
+}
+
+// Purgeable allows purging the entire database by undoing all migrations.
+// This is deliberately non-straightforward to use to prevent accidental purging.
+// Call .Purge() on the instance to do the thing.
+type Purgeable struct{ *sql.DB }
+
+func (p Purgeable) Purge() error {
+	source, err := iofs.New(migrations.FS, ".")
+	if err != nil {
+		return err
+	}
+
+	driver, err := postgres.WithInstance(p.DB, &postgres.Config{})
+	if err != nil {
+		return err
+	}
+
+	m, err := migrate.NewWithInstance("iofs", source, "postgres", driver)
+	if err != nil {
+		return err
+	}
+
+	return m.Down()
+}

cmd/feedizer/main.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"git.zom.bi/fanir/feedizer/cmd/feedizer/config"
+	"git.zom.bi/fanir/feedizer/cmd/feedizer/database"
+	"github.com/golang-migrate/migrate/v4"
+)
+
+func main() {
+	if os.Geteuid() == 0 {
+		log.Fatalln("do not run feedizer as root")
+	}
+
+	configFile := flag.String("C", "", "custom config file path")
+	autoMigrate := flag.Bool("m", true, "Automatically migrate the database to the highest version.")
+	flag.Parse()
+
+	switch flag.Arg(0) {
+	case "", "run":
+		startApp(*autoMigrate)
+
+	case "newconfig":
+		path, err := config.WriteFile(*configFile)
+		if err != nil {
+			fmt.Printf("cannot write config file %s: %s", path, err)
+			os.Exit(1)
+		}
+		fmt.Println("wrote config file", path)
+
+	case "purgedb":
+		purgeDB()
+
+	default:
+		fmt.Printf("invalid action %s\n\n", flag.Arg(0))
+		fmt.Println("valid actions are probably: run (default), newconfig, purgedb")
+	}
+}
+
+func startApp(automigrate bool) {
+	cfg, err := config.Load()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	db, err := database.Open(cfg.Database)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	if automigrate {
+		log.Println("migrating database...")
+		if err = database.Migrate(db); err != nil && !errors.Is(err, migrate.ErrNoChange) {
+			log.Fatalln(err)
+		}
+		log.Println("migration successful")
+	}
+}
+
+func purgeDB() {
+	cfg, err := config.Load()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	db, err := database.Open(cfg.Database)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	fmt.Printf("PURGE database %s? You will loose all data. [yes/no]\n", database.LogName)
+	in, err := bufio.NewReader(os.Stdin).ReadString('\n')
+	if err != nil {
+		log.Fatalln(err)
+	}
+	if in != "yes\n" {
+		fmt.Println("Aborting. (You must type \"yes\" to continue)")
+		return
+	}
+
+	fmt.Println("purging database...")
+	if err = (database.Purgeable{DB: db}).Purge(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
+		log.Fatalln(err)
+	}
+	fmt.Println("purging successful")
+}

go.mod

@@ -0,0 +1,21 @@
+module git.zom.bi/fanir/feedizer
+
+go 1.17
+
+require (
+	github.com/adrg/xdg v0.4.0
+	github.com/golang-migrate/migrate/v4 v4.15.1
+	github.com/google/uuid v1.3.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+)
+
+require (
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/lib/pq v1.10.4 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
+	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
+	google.golang.org/grpc v1.43.0 // indirect
+)

models/migrations/migrations.go

@@ -0,0 +1,8 @@
+package migrations
+
+import (
+	"embed"
+)
+
+//go:embed *.sql
+var FS embed.FS

models/models.go

@@ -0,0 +1,55 @@
+package models
+
+import (
+	"net"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+type Feed struct {
+	ID              uuid.UUID // PRIMARY KEY
+	Slug            string    // NOT NULL UNIQUE
+	URI             string    // NOT NULL
+	AutoRefresh     bool      // NOT NULL
+	RefreshInterval int       //
+	NextRefresh     time.Time // NOT NULL
+	Expire          bool      // NOT NULL
+	ExpireDate      time.Time //
+	Password        string    //
+	CreationIP      net.IP    // NOT NULL
+	CreationDate    time.Time // NOT NULL DEFAULT now()
+}
+
+type Feeditem struct {
+	Feed      int       // NOT NULL REFERENCES feed ON DELETE CASCADE ON UPDATE CASCADE
+	Timestamp time.Time // NOT NULL DEFAULT now()
+	HTML      string    // NOT NULL
+	Diff      string    //
+	//PRIMARY KEY (feed_ time.Time //)
+}
+
+type Announcement struct {
+	ID              uuid.UUID // PRIMARY KEY
+	Title           string    // NOT NULL
+	Content         string    // NOT NULL
+	Abstract        string    //
+	PublicationDate time.Time // NOT NULL DEFAULT now()
+	ShowUntil       time.Time //
+	IsImportant     bool      // NOT NULL
+}
+
+type Feedhistory struct {
+	Feed            int       // NOT NULL REFERENCES feed ON DELETE CASCADE ON UPDATE CASCADE
+	Timestamp       time.Time // NOT NULL DEFAULT now()
+	IP              net.IP    // NOT NULL
+	Slug            string    //
+	URI             string    //
+	AutoRefresh     bool      //
+	RefreshInterval int       //
+	NextRefresh     time.Time //
+	Expire          bool      // NOT NULL
+	ExpireDate      time.Time //
+	Password        string    //
+	//PRIMARY KEY (feed_ time.Time //)
+}