feat: Initial norwegian bluesky feed generator

Subscribe to Bluesky firehose and get all posts written in Norwegian
bokmål, nynorsk, and sami. Written in Golang.

Signed-off-by: Snorre Magnus Davøen <[email protected]>

.gitignore

@@ -0,0 +1,2 @@
+
+dist/

.goreleaser.yaml

@@ -0,0 +1,88 @@
+# This is an example .goreleaser.yml file with some sensible defaults.
+# Make sure to check the documentation at https://goreleaser.com
+
+# The lines bellow are called `modelines`. See `:help modeline`
+# Feel free to remove those if you don't want/need to use them.
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj
+
+before:
+  hooks:
+    # You may remove this if you don't use go modules.
+    - go mod tidy
+    # you may remove this if you don't need go generate
+    - go generate ./...
+
+builds:
+  - env:
+      - CGO_ENABLED=0
+    flags:
+      - -o
+    goos:
+      - linux
+      - windows
+      - darwin
+
+
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of `uname`.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+      - goos: windows
+        format: zip
+
+
+dockers:
+- image_templates: ["ghcr.io/snorremd/{{ .ProjectName }}:{{ .Version }}-amd64"]
+  dockerfile: Dockerfile
+  use: buildx
+  build_flag_templates:
+  - --platform=linux/amd64
+  - --label=org.opencontainers.image.title={{ .ProjectName }}
+  - --label=org.opencontainers.image.description={{ .ProjectName }}
+  - --label=org.opencontainers.image.url=https://github.com/caarlos0/{{ .ProjectName }}
+  - --label=org.opencontainers.image.source=https://github.com/caarlos0/{{ .ProjectName }}
+  - --label=org.opencontainers.image.version={{ .Version }}
+  - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }}
+  - --label=org.opencontainers.image.revision={{ .FullCommit }}
+  - --label=org.opencontainers.image.licenses=MIT
+- image_templates: ["ghcr.io/snorremd/{{ .ProjectName }}:{{ .Version }}-arm64v8"]
+  goarch: arm64
+  dockerfile: Dockerfile
+  use: buildx
+  build_flag_templates:
+  - --platform=linux/arm64/v8
+  - --label=org.opencontainers.image.title={{ .ProjectName }}
+  - --label=org.opencontainers.image.description={{ .ProjectName }}
+  - --label=org.opencontainers.image.url=https://github.com/caarlos0/{{ .ProjectName }}
+  - --label=org.opencontainers.image.source=https://github.com/caarlos0/{{ .ProjectName }}
+  - --label=org.opencontainers.image.version={{ .Version }}
+  - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }}
+  - --label=org.opencontainers.image.revision={{ .FullCommit }}
+  - --label=org.opencontainers.image.licenses=MIT
+
+
+docker_manifests:
+- name_template: ghcr.io/snorremd/{{ .ProjectName }}:{{ .Version }}
+  image_templates:
+  - ghcr.io/snorremd/{{ .ProjectName }}:{{ .Version }}-amd64
+  - ghcr.io/snorremd/{{ .ProjectName }}:{{ .Version }}-arm64v8
+- name_template: ghcr.io/caarlos0/{{ .ProjectName }}:latest
+  image_templates:
+  - ghcr.io/snorremd/{{ .ProjectName }}:{{ .Version }}-amd64
+  - ghcr.io/snorremd/{{ .ProjectName }}:{{ .Version }}-arm64v8
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - "^docs:"
+      - "^test:"

Dockerfile

@@ -0,0 +1,4 @@
+FROM scratch
+ENTRYPOINT ["/norsky"]
+CMD ["serve"]
+COPY norsky /

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Snorre Magnus Davøen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,83 @@
+# Norsky
+
+Norsky is a lightweight ATProto feed server that serves Norwegian posts from the Bluesky instance.
+It is written in Go and stores posts in a SQLite database to keep things simple.
+
+## Installation
+
+The feed server is a standalone go binary that you can run on your machine.
+It is also available as a Docker image.
+
+
+### From GitHub release
+
+```bash
+curl -L 
+```
+
+### Go install
+
+```bash
+go install github.com/snorreio/norsky
+```
+
+### Docker
+
+```bash
+docker run -d --name norsky -p 8080:8080 -v /path/to/db:/db ghcr.io/snorreio/norsky:latest
+```
+
+
+## Development
+
+The application has been developed using go 1.21.1.
+You should at least have go 1.18+ installed to build the application.
+
+
+To get started clone the repository and run go run on the main file to list the available commands.
+
+```bash
+go run main.go --help
+```
+
+The application uses the [cobra](https://github.com/spf13/cobra) library to manage commands and flags.
+[cobra-cli](https://github.com/spf13/cobra-cli) should be used to generate new commands.
+
+### Structure
+
+The application follows the standard way to structure a Go project and contains the following packages:
+
+- `main` - The main package that contains the entrypoint for the application.
+- `cmd` - The command package that contains the different commands for the application using cobra.
+- `server` - The server package that contains setup code for initializing a Fiber server.
+- `database` - The database package that contains the database connection, models, migrations, and queries.
+- `firehose` - The firehose package that contains the firehose client and the firehose subscription.
+
+```
+.
+├── cmd
+│   ├── gc.go
+│   ├── root.go
+│   ├── serve.go
+│   └── subscribe.go
+├── database
+│   └── database.go
+├── firehose
+│   └── firehose.go
+├── server
+│   └── server.go
+├── go.mod
+├── go.sum
+├── LICENSE
+├── main.go
+├── norsky
+├── README.md
+```
+
+## Release
+
+This project uses [goreleaser](https://goreleaser.com/) to build and release the application.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

cmd/gc.go

@@ -0,0 +1,36 @@
+/*
+Copyright © 2023 NAME HERE <EMAIL ADDRESS>
+*/
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// gcCmd represents the gc command
+var gcCmd = &cobra.Command{
+	Use:   "gc",
+	Short: "Garbage collect the SQLite database",
+	Long: `Delete all posts that are older than 30 days from the SQLite database.
+
+Can be run as a cron job to keep the database size down.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("gc called")
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(gcCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// gcCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// gcCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

cmd/root.go

@@ -0,0 +1,47 @@
+/*
+Copyright © 2023 NAME HERE <EMAIL ADDRESS>
+*/
+package cmd
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "norsky",
+	Short: "A Bluesky feed for Norwegian Bluesky posts",
+	Long: `A Bluesky feed that only shows posts written in
+Norwegian bokmål, nynorsk and sami.
+
+Norsky works by subscribing to the Bluesky firehose and filtering
+down the result to posts written in Norwegian bokmål, nynorsk and sami.
+The result is written to an SQLite database and can be accessed via an
+HTTP API as specified in the Bluesky API specification.`,
+	// Uncomment the following line if your bare application
+	// has an action associated with it:
+	// Run: func(cmd *cobra.Command, args []string) { },
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}
+
+func init() {
+	// Here you will define your flags and configuration settings.
+	// Cobra supports persistent flags, which, if defined here,
+	// will be global for your application.
+
+	// rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.norsky.yaml)")
+
+	// Cobra also supports local flags, which will only run
+	// when this action is called directly.
+	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

cmd/serve.go

@@ -0,0 +1,96 @@
+/*
+Copyright © 2023 NAME HERE <EMAIL ADDRESS>
+*/
+package cmd
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/signal"
+	"sync"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"norsky/firehose"
+	"norsky/server"
+)
+
+// serveCmd represents the serve command
+var serveCmd = &cobra.Command{
+	Use:   "serve",
+	Short: "Serve the norsky feed",
+	Long: `Starts the norsky feed HTTP server and firehose subscriber.
+
+Launches the HTTP server on the specified or default port and subscribes to the
+Bluesky firehose. All posts written in Norwegian bokmål, nynorsk and sami are
+written to the SQLite database and can be accessed via the HTTP API as specified
+in the Bluesky API specification.`,
+	Run: func(cmd *cobra.Command, args []string) {
+
+		fmt.Println("Starting norsky...")
+
+		// Channel for subscribing to bluesky posts
+		postChan := make(chan interface{})
+
+		// Setup the server and firehose
+		app := server.Server()
+		fh := firehose.New(postChan)
+
+		// Graceful shutdown
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, os.Interrupt)
+		var wg sync.WaitGroup
+
+		go func() {
+			<-c
+			fmt.Println("Gracefully shutting down...")
+			app.ShutdownWithTimeout(60 * time.Second)
+			defer wg.Add(-2) // Decrement the waitgroup counter by 2 after shutdown of server and firehose
+			fh.Shutdown()
+		}()
+
+		go func() {
+			fmt.Println("Subscribing to firehose...")
+			if err := fh.Subscribe(); err != nil {
+				log.Panic(err)
+			}
+		}()
+
+		go func() {
+			fmt.Println("Starting server...")
+			if err := app.Listen(":3000"); err != nil {
+				log.Panic(err)
+			}
+		}()
+
+		go func() {
+			// Subscribe to the post channel and log the posts
+			for post := range postChan {
+				fmt.Println(post)
+			}
+		}()
+
+		// Wait for both the server and firehose to shutdown
+		wg.Add(2)
+		wg.Wait()
+
+		fmt.Println("Done!")
+
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(serveCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// serveCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// serveCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}