diff --git a/README.md b/README.md index f41a2c0..d9c795a 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ Each feed is defined in a `[[feeds]]` section and currently requires the followi - `avatar_path` - The path to the avatar image for the feed. - `languages` - The languages (iso-639-1 codes) supported by the feed. - `keywords` - The keywords to filter posts by. +- `exclude_replies` - Whether to exclude replies from the feed. If you want to run a german language feed you can add the following to your `feeds.toml` file: diff --git a/config/config.go b/config/config.go index 9ac13bd..1b26baa 100644 --- a/config/config.go +++ b/config/config.go @@ -8,12 +8,13 @@ import ( ) type FeedConfig struct { - ID string `toml:"id"` - DisplayName string `toml:"display_name"` - Description string `toml:"description"` - AvatarPath string `toml:"avatar_path,omitempty"` - Languages []string `toml:"languages"` - Keywords []string `toml:"keywords,omitempty"` + ID string `toml:"id"` + DisplayName string `toml:"display_name"` + Description string `toml:"description"` + AvatarPath string `toml:"avatar_path"` + Languages []string `toml:"languages"` + Keywords []string `toml:"keywords"` + ExcludeReplies bool `toml:"exclude_replies,omitempty" default:"false"` } type Config struct { diff --git a/db/migrations/20250118190737_add_parent_uri.down.sqlite b/db/migrations/20250118190737_add_parent_uri.down.sqlite new file mode 100644 index 0000000..c5d416c --- /dev/null +++ b/db/migrations/20250118190737_add_parent_uri.down.sqlite @@ -0,0 +1 @@ +ALTER TABLE posts DROP COLUMN parent_uri; \ No newline at end of file diff --git a/db/migrations/20250118190737_add_parent_uri.up.sqlite b/db/migrations/20250118190737_add_parent_uri.up.sqlite new file mode 100644 index 0000000..5590113 --- /dev/null +++ b/db/migrations/20250118190737_add_parent_uri.up.sqlite @@ -0,0 +1 @@ +ALTER TABLE posts ADD COLUMN parent_uri TEXT; \ No newline at end of file diff --git a/db/reader.go b/db/reader.go index 566f959..d4919ad 100644 --- a/db/reader.go +++ b/db/reader.go @@ -46,7 +46,7 @@ func NewReader(database string) *Reader { } } -func (reader *Reader) GetFeed(langs []string, queries []string, limit int, postId int64) ([]models.FeedPost, error) { +func (reader *Reader) GetFeed(langs []string, queries []string, limit int, postId int64, excludeReplies bool) ([]models.FeedPost, error) { sb := sqlbuilder.NewSelectBuilder() sb.Select("DISTINCT posts.id", "posts.uri").From("posts") @@ -54,6 +54,11 @@ func (reader *Reader) GetFeed(langs []string, queries []string, limit int, postI sb.Where(sb.LessThan("posts.id", postId)) } + // Add condition to exclude replies if requested + if excludeReplies { + sb.Where(sb.IsNull("parent_uri")) + } + // Build language conditions if specified if len(langs) > 0 { sb.Join("post_languages", "posts.id = post_languages.post_id") diff --git a/db/writer.go b/db/writer.go index e7fa8cb..73ad1e4 100644 --- a/db/writer.go +++ b/db/writer.go @@ -97,6 +97,7 @@ func createPost(ctx context.Context, db *sql.DB, post models.Post) (int64, error log.WithFields(log.Fields{ "uri": post.Uri, + "parent_uri": post.ParentUri, "languages": post.Languages, "created_at": time.Unix(post.CreatedAt, 0).Format(time.RFC3339), // Record lag from when the post was created to when it was processed @@ -105,8 +106,8 @@ func createPost(ctx context.Context, db *sql.DB, post models.Post) (int64, error // Post insert query insertPost := sqlbuilder.NewInsertBuilder() - insertPost.InsertInto("posts").Cols("uri", "created_at", "text") - insertPost.Values(post.Uri, post.CreatedAt, post.Text) + insertPost.InsertInto("posts").Cols("uri", "created_at", "text", "parent_uri") + insertPost.Values(post.Uri, post.CreatedAt, post.Text, post.ParentUri) sql, args := insertPost.Build() diff --git a/feeds/feeds.go b/feeds/feeds.go index 47fa8bd..34922bf 100644 --- a/feeds/feeds.go +++ b/feeds/feeds.go @@ -13,10 +13,10 @@ import ( type Algorithm func(reader *db.Reader, cursor string, limit int) (*models.FeedResponse, error) // Reuse genericAlgo for all algorithms -func genericAlgo(reader *db.Reader, cursor string, limit int, languages []string, keywords []string) (*models.FeedResponse, error) { +func genericAlgo(reader *db.Reader, cursor string, limit int, languages []string, keywords []string, excludeReplies bool) (*models.FeedResponse, error) { postId := safeParseCursor(cursor) - posts, err := reader.GetFeed(languages, keywords, limit+1, postId) + posts, err := reader.GetFeed(languages, keywords, limit+1, postId, excludeReplies) if err != nil { log.Error("Error getting feed", err) return nil, err @@ -55,13 +55,14 @@ func safeParseCursor(cursor string) int64 { } type Feed struct { - Id string - DisplayName string - Description string - AvatarPath string - Languages []string - Keywords []string - Algorithm Algorithm + Id string + DisplayName string + Description string + AvatarPath string + Languages []string + Keywords []string + ExcludeReplies bool + Algorithm Algorithm } // Create a new function to initialize feeds from config @@ -70,13 +71,14 @@ func InitializeFeeds(cfg *config.Config) map[string]Feed { for _, feedCfg := range cfg.Feeds { feeds[feedCfg.ID] = Feed{ - Id: feedCfg.ID, - DisplayName: feedCfg.DisplayName, - Description: feedCfg.Description, - AvatarPath: feedCfg.AvatarPath, - Languages: feedCfg.Languages, - Keywords: feedCfg.Keywords, - Algorithm: createAlgorithm(feedCfg.Languages, feedCfg.Keywords), + Id: feedCfg.ID, + DisplayName: feedCfg.DisplayName, + Description: feedCfg.Description, + AvatarPath: feedCfg.AvatarPath, + Languages: feedCfg.Languages, + Keywords: feedCfg.Keywords, + ExcludeReplies: feedCfg.ExcludeReplies, + Algorithm: createAlgorithm(feedCfg.Languages, feedCfg.Keywords, feedCfg.ExcludeReplies), } } @@ -84,8 +86,8 @@ func InitializeFeeds(cfg *config.Config) map[string]Feed { } // Helper function to create an algorithm based on languages -func createAlgorithm(languages []string, keywords []string) Algorithm { +func createAlgorithm(languages []string, keywords []string, excludeReplies bool) Algorithm { return func(reader *db.Reader, cursor string, limit int) (*models.FeedResponse, error) { - return genericAlgo(reader, cursor, limit, languages, keywords) + return genericAlgo(reader, cursor, limit, languages, keywords, excludeReplies) } } diff --git a/firehose/firehose.go b/firehose/firehose.go index 179d2a6..884dd1e 100644 --- a/firehose/firehose.go +++ b/firehose/firehose.go @@ -553,12 +553,19 @@ func (p *PostProcessor) processPost(evt *atproto.SyncSubscribeRepos_Commit, op * return fmt.Errorf("failed to parse creation time: %w", err) } + // Extract parent URI if this is a reply + var parentUri string + if record.Reply != nil && record.Reply.Parent != nil { + parentUri = record.Reply.Parent.Uri + } + p.postChan <- models.CreatePostEvent{ Post: models.Post{ Uri: uri, CreatedAt: createdAt.Unix(), Text: record.Text, Languages: langs, + ParentUri: parentUri, }, } diff --git a/models/models.go b/models/models.go index 29f8452..b89e1de 100644 --- a/models/models.go +++ b/models/models.go @@ -9,6 +9,7 @@ type Post struct { Text string `json:"text"` Languages []string `json:"languages"` Uri string `json:"uri"` + ParentUri string `json:"parentUri,omitempty"` } // Omit all but the Uri field