feat: Allow excluding replies in feed config

For some feeds one might want to limit the posts to original posts (i.e.
posts that are not replies). In feeds like Fjernsyn this will help keep
the feed more relevant and focused.

README.md

@@ -116,6 +116,7 @@ Each feed is defined in a `[[feeds]]` section and currently requires the followi
 - `avatar_path` - The path to the avatar image for the feed.
 - `languages` - The languages (iso-639-1 codes) supported by the feed.
 - `keywords` - The keywords to filter posts by.
+- `exclude_replies` - Whether to exclude replies from the feed.
 
 If you want to run a german language feed you can add the following to your `feeds.toml` file:
 

config/config.go

@@ -8,12 +8,13 @@ import (
 )
 
 type FeedConfig struct {
-	ID          string   `toml:"id"`
-	DisplayName string   `toml:"display_name"`
-	Description string   `toml:"description"`
-	AvatarPath  string   `toml:"avatar_path,omitempty"`
-	Languages   []string `toml:"languages"`
-	Keywords    []string `toml:"keywords,omitempty"`
+	ID             string   `toml:"id"`
+	DisplayName    string   `toml:"display_name"`
+	Description    string   `toml:"description"`
+	AvatarPath     string   `toml:"avatar_path"`
+	Languages      []string `toml:"languages"`
+	Keywords       []string `toml:"keywords"`
+	ExcludeReplies bool     `toml:"exclude_replies,omitempty" default:"false"`
 }
 
 type Config struct {

db/migrations/20250118190737_add_parent_uri.down.sqlite

@@ -0,0 +1 @@
+ALTER TABLE posts DROP COLUMN parent_uri;

db/migrations/20250118190737_add_parent_uri.up.sqlite

@@ -0,0 +1 @@
+ALTER TABLE posts ADD COLUMN parent_uri TEXT;

db/reader.go

@@ -46,14 +46,19 @@ func NewReader(database string) *Reader {
 	}
 }
 
-func (reader *Reader) GetFeed(langs []string, queries []string, limit int, postId int64) ([]models.FeedPost, error) {
+func (reader *Reader) GetFeed(langs []string, queries []string, limit int, postId int64, excludeReplies bool) ([]models.FeedPost, error) {
 	sb := sqlbuilder.NewSelectBuilder()
 	sb.Select("DISTINCT posts.id", "posts.uri").From("posts")
 
 	if postId != 0 {
 		sb.Where(sb.LessThan("posts.id", postId))
 	}
 
+	// Add condition to exclude replies if requested
+	if excludeReplies {
+		sb.Where(sb.IsNull("parent_uri"))
+	}
+
 	// Build language conditions if specified
 	if len(langs) > 0 {
 		sb.Join("post_languages", "posts.id = post_languages.post_id")

db/writer.go

@@ -97,6 +97,7 @@ func createPost(ctx context.Context, db *sql.DB, post models.Post) (int64, error
 
 	log.WithFields(log.Fields{
 		"uri":        post.Uri,
+		"parent_uri": post.ParentUri,
 		"languages":  post.Languages,
 		"created_at": time.Unix(post.CreatedAt, 0).Format(time.RFC3339),
 		// Record lag from when the post was created to when it was processed
@@ -105,8 +106,8 @@ func createPost(ctx context.Context, db *sql.DB, post models.Post) (int64, error
 
 	// Post insert query
 	insertPost := sqlbuilder.NewInsertBuilder()
-	insertPost.InsertInto("posts").Cols("uri", "created_at", "text")
-	insertPost.Values(post.Uri, post.CreatedAt, post.Text)
+	insertPost.InsertInto("posts").Cols("uri", "created_at", "text", "parent_uri")
+	insertPost.Values(post.Uri, post.CreatedAt, post.Text, post.ParentUri)
 
 	sql, args := insertPost.Build()
 

feeds/feeds.go

@@ -13,10 +13,10 @@ import (
 type Algorithm func(reader *db.Reader, cursor string, limit int) (*models.FeedResponse, error)
 
 // Reuse genericAlgo for all algorithms
-func genericAlgo(reader *db.Reader, cursor string, limit int, languages []string, keywords []string) (*models.FeedResponse, error) {
+func genericAlgo(reader *db.Reader, cursor string, limit int, languages []string, keywords []string, excludeReplies bool) (*models.FeedResponse, error) {
 	postId := safeParseCursor(cursor)
 
-	posts, err := reader.GetFeed(languages, keywords, limit+1, postId)
+	posts, err := reader.GetFeed(languages, keywords, limit+1, postId, excludeReplies)
 	if err != nil {
 		log.Error("Error getting feed", err)
 		return nil, err
@@ -55,13 +55,14 @@ func safeParseCursor(cursor string) int64 {
 }
 
 type Feed struct {
-	Id          string
-	DisplayName string
-	Description string
-	AvatarPath  string
-	Languages   []string
-	Keywords    []string
-	Algorithm   Algorithm
+	Id             string
+	DisplayName    string
+	Description    string
+	AvatarPath     string
+	Languages      []string
+	Keywords       []string
+	ExcludeReplies bool
+	Algorithm      Algorithm
 }
 
 // Create a new function to initialize feeds from config
@@ -70,22 +71,23 @@ func InitializeFeeds(cfg *config.Config) map[string]Feed {
 
 	for _, feedCfg := range cfg.Feeds {
 		feeds[feedCfg.ID] = Feed{
-			Id:          feedCfg.ID,
-			DisplayName: feedCfg.DisplayName,
-			Description: feedCfg.Description,
-			AvatarPath:  feedCfg.AvatarPath,
-			Languages:   feedCfg.Languages,
-			Keywords:    feedCfg.Keywords,
-			Algorithm:   createAlgorithm(feedCfg.Languages, feedCfg.Keywords),
+			Id:             feedCfg.ID,
+			DisplayName:    feedCfg.DisplayName,
+			Description:    feedCfg.Description,
+			AvatarPath:     feedCfg.AvatarPath,
+			Languages:      feedCfg.Languages,
+			Keywords:       feedCfg.Keywords,
+			ExcludeReplies: feedCfg.ExcludeReplies,
+			Algorithm:      createAlgorithm(feedCfg.Languages, feedCfg.Keywords, feedCfg.ExcludeReplies),
 		}
 	}
 
 	return feeds
 }
 
 // Helper function to create an algorithm based on languages
-func createAlgorithm(languages []string, keywords []string) Algorithm {
+func createAlgorithm(languages []string, keywords []string, excludeReplies bool) Algorithm {
 	return func(reader *db.Reader, cursor string, limit int) (*models.FeedResponse, error) {
-		return genericAlgo(reader, cursor, limit, languages, keywords)
+		return genericAlgo(reader, cursor, limit, languages, keywords, excludeReplies)
 	}
 }

firehose/firehose.go

@@ -553,12 +553,19 @@ func (p *PostProcessor) processPost(evt *atproto.SyncSubscribeRepos_Commit, op *
 		return fmt.Errorf("failed to parse creation time: %w", err)
 	}
 
+	// Extract parent URI if this is a reply
+	var parentUri string
+	if record.Reply != nil && record.Reply.Parent != nil {
+		parentUri = record.Reply.Parent.Uri
+	}
+
 	p.postChan <- models.CreatePostEvent{
 		Post: models.Post{
 			Uri:       uri,
 			CreatedAt: createdAt.Unix(),
 			Text:      record.Text,
 			Languages: langs,
+			ParentUri: parentUri,
 		},
 	}
 

models/models.go

@@ -9,6 +9,7 @@ type Post struct {
 	Text      string   `json:"text"`
 	Languages []string `json:"languages"`
 	Uri       string   `json:"uri"`
+	ParentUri string   `json:"parentUri,omitempty"`
 }
 
 // Omit all but the Uri field