Add a new field time to struct Post

Cargo.toml

@@ -7,6 +7,7 @@ publish = false
 [dependencies]
 anyhow = "1.0.89"
 bytes = "1.7.2"
+chrono = "0.4.39"
 clap = { version = "4.5.19", features = ["derive"] }
 futures = "0.3.31"
 headless_chrome = "1.0.15"

src/source/mod.rs

@@ -5,6 +5,7 @@ pub mod platform;
 use std::{fmt, fmt::Display, future::Future, pin::Pin, slice, vec};
 
 use anyhow::ensure;
+use chrono::{DateTime, Local};
 pub use content::*;
 
 use crate::platform::{PlatformMetadata, PlatformTrait};
@@ -146,6 +147,7 @@ pub struct Post {
     pub user: Option<User>,
     pub content: PostContent,
     urls: PostUrls,
+    pub time: DateTime<Local>,
     pub repost_from: Option<RepostFrom>,
     attachments: Vec<PostAttachment>,
 }
@@ -503,6 +505,7 @@ mod tests {
                 user: None,
                 content: PostContent::plain("content1"),
                 urls: PostUrls::new(PostUrl::Identity("id1".into())),
+                time: DateTime::UNIX_EPOCH.into(),
                 repost_from: None,
                 attachments: vec![],
             }])),
@@ -525,13 +528,15 @@ mod tests {
                     user: None,
                     content: PostContent::plain("content1"),
                     urls: PostUrls::new(PostUrl::Identity("id1".into())),
+                    time: DateTime::UNIX_EPOCH.into(),
                     repost_from: None,
                     attachments: vec![],
                 },
                 Post {
                     user: None,
                     content: PostContent::plain("content2"),
                     urls: PostUrls::new(PostUrl::Identity("id2".into())),
+                    time: DateTime::UNIX_EPOCH.into(),
                     repost_from: None,
                     attachments: vec![],
                 },
@@ -549,13 +554,15 @@ mod tests {
                     user: None,
                     content: PostContent::plain("content1"),
                     urls: PostUrls::new(PostUrl::Identity("id1".into())),
+                    time: DateTime::UNIX_EPOCH.into(),
                     repost_from: None,
                     attachments: vec![],
                 },
                 Post {
                     user: None,
                     content: PostContent::plain("content2"),
                     urls: PostUrls::new(PostUrl::Identity("id2".into())),
+                    time: DateTime::UNIX_EPOCH.into(),
                     repost_from: None,
                     attachments: vec![],
                 },
@@ -588,6 +595,7 @@ mod tests {
                 user: None,
                 content: PostContent::plain("content3"),
                 urls: PostUrls::new(PostUrl::Identity("id3".into())),
+                time: DateTime::UNIX_EPOCH.into(),
                 repost_from: None,
                 attachments: vec![],
             }])),
@@ -607,20 +615,23 @@ mod tests {
                     user: None,
                     content: PostContent::plain("content1"),
                     urls: PostUrls::new(PostUrl::Identity("id1".into())),
+                    time: DateTime::UNIX_EPOCH.into(),
                     repost_from: None,
                     attachments: vec![],
                 },
                 Post {
                     user: None,
                     content: PostContent::plain("content2"),
                     urls: PostUrls::new(PostUrl::Identity("id2".into())),
+                    time: DateTime::UNIX_EPOCH.into(),
                     repost_from: None,
                     attachments: vec![],
                 },
                 Post {
                     user: None,
                     content: PostContent::plain("content3"),
                     urls: PostUrls::new(PostUrl::Identity("id3".into())),
+                    time: DateTime::UNIX_EPOCH.into(),
                     repost_from: None,
                     attachments: vec![],
                 },
@@ -673,6 +684,7 @@ mod tests {
                 user: None,
                 content: PostContent::plain("content1"),
                 urls: PostUrls::new(PostUrl::Identity("id1".into())),
+                time: DateTime::UNIX_EPOCH.into(),
                 repost_from: None,
                 attachments: vec![],
             }])),

src/source/platform/bilibili/space.rs

@@ -1,6 +1,7 @@
 use std::{collections::HashSet, fmt, fmt::Display, future::Future, ops::DerefMut, pin::Pin};
 
-use anyhow::{anyhow, bail, Ok};
+use anyhow::{anyhow, bail, ensure};
+use chrono::DateTime;
 use once_cell::sync::Lazy;
 use reqwest::header::{self, HeaderValue};
 use serde::Deserialize;
@@ -81,6 +82,16 @@ mod data {
         Pgc(ModuleAuthorPgc),
     }
 
+    impl ModuleAuthor {
+        pub fn pub_time(&self) -> Option<u64> {
+            let pub_ts = match self {
+                ModuleAuthor::Normal(normal) => normal.pub_ts,
+                ModuleAuthor::Pgc(pgc) => pgc.pub_ts,
+            };
+            (pub_ts != 0).then_some(pub_ts)
+        }
+    }
+
     impl From<ModuleAuthor> for User {
         fn from(value: ModuleAuthor) -> Self {
             match value {
@@ -111,6 +122,7 @@ mod data {
         pub face: String, // URL
         pub mid: u64,
         pub name: String,
+        pub pub_ts: u64, // Always 0?
     }
 
     #[derive(Debug, Deserialize)]
@@ -644,10 +656,24 @@ fn parse_response(resp: data::SpaceHistory, blocked: &mut BlockedPostIds) -> any
             })
             .unwrap_or(major_url);
 
+        let time = item
+            .modules
+            .author
+            .pub_time()
+            .or(parent_item.and_then(|p| p.modules.author.pub_time()));
+        ensure!(
+            time.is_some(),
+            "bilibili space found a post with no time: '{item:?}'"
+        );
+        let time = DateTime::from_timestamp(time.unwrap() as i64, 0)
+            .ok_or_else(|| anyhow!("invalid pub time, url={url:?}"))?
+            .into();
+
         Ok(Post {
             user: Some(item.modules.author.clone().into()),
             content,
             urls: PostUrls::new(url),
+            time,
             repost_from: original.map(|original| RepostFrom::Recursion(Box::new(original))),
             attachments: item
                 .modules

src/source/platform/bilibili/video.rs

@@ -1,6 +1,7 @@
 use std::{fmt, future::Future, pin::Pin};
 
 use anyhow::{anyhow, ensure};
+use chrono::DateTime;
 use serde::Deserialize;
 use serde_json as json;
 
@@ -42,6 +43,8 @@ mod data {
     pub struct Archive {
         pub aid: u64,
         pub title: String,
+        pub pubdate: u64,
+        pub ctime: u64,
         pub pic: String, // Image URL
         pub bvid: String,
     }
@@ -120,21 +123,27 @@ fn parse_response(resp: data::SeriesArchives) -> anyhow::Result<Posts> {
     let videos = resp
         .archives
         .into_iter()
-        .map(|archive| Post {
-            user: None,
-            content: PostContent::plain(archive.title),
-            urls: PostUrl::new_clickable(
-                format!("https://www.bilibili.com/video/{}", archive.bvid),
-                "查看视频",
-            )
-            .into(),
-            repost_from: None,
-            attachments: vec![PostAttachment::Image(PostAttachmentImage {
-                media_url: upgrade_to_https(&archive.pic),
-                has_spoiler: false,
-            })],
+        .map(|archive| -> anyhow::Result<Post> {
+            let time = DateTime::from_timestamp(archive.ctime as i64, 0)
+                .ok_or_else(|| anyhow!("invalid ctime {}, aid={}", archive.ctime, archive.aid))?
+                .into();
+            Ok(Post {
+                user: None,
+                content: PostContent::plain(archive.title),
+                urls: PostUrl::new_clickable(
+                    format!("https://www.bilibili.com/video/{}", archive.bvid),
+                    "查看视频",
+                )
+                .into(),
+                time,
+                repost_from: None,
+                attachments: vec![PostAttachment::Image(PostAttachmentImage {
+                    media_url: upgrade_to_https(&archive.pic),
+                    has_spoiler: false,
+                })],
+            })
         })
-        .collect();
+        .collect::<Result<_, _>>()?;
 
     Ok(Posts(videos))
 }

src/source/platform/twitter/mod.rs

@@ -8,6 +8,7 @@ use std::{
 };
 
 use anyhow::{anyhow, bail};
+use chrono::DateTime;
 use request::*;
 use serde::Deserialize;
 use spdlog::prelude::*;
@@ -463,13 +464,13 @@ impl FetcherInner {
             })
             .map(|result| result.result.into_tweet())
             .map(parse_tweet)
-            .collect::<Vec<_>>();
+            .collect::<Result<Vec<_>, _>>()?;
 
         Ok(Posts(posts))
     }
 }
 
-fn parse_tweet(tweet: data::Tweet) -> Post {
+fn parse_tweet(tweet: data::Tweet) -> anyhow::Result<Post> {
     let content = if tweet.legacy.retweeted_status_result.is_none() {
         Some(replace_entities(
             tweet.legacy.full_text,
@@ -492,7 +493,12 @@ fn parse_tweet(tweet: data::Tweet) -> Post {
     } else {
         tweet.quoted_status_result.and_then(|q| q.into_option())
     }
-    .map(|result| RepostFrom::Recursion(Box::new(parse_tweet(result.result.into_tweet()))));
+    .map(|result| -> anyhow::Result<RepostFrom> {
+        Ok(RepostFrom::Recursion(Box::new(parse_tweet(
+            result.result.into_tweet(),
+        )?)))
+    })
+    .transpose()?;
 
     let possibly_sensitive = tweet.legacy.possibly_sensitive.unwrap_or(false);
 
@@ -591,13 +597,23 @@ fn parse_tweet(tweet: data::Tweet) -> Post {
         })
         .collect();
 
-    Post {
+    let time = DateTime::parse_from_str(&tweet.legacy.created_at, "%a %b %d %H:%M:%S %z %Y")
+        .map_err(|err| {
+            anyhow!(
+                "failed to parse tweet time '{}', err: {err}, urls={urls:?}",
+                tweet.legacy.created_at
+            )
+        })?
+        .into();
+
+    Ok(Post {
         user: Some(tweet.core.user_results.result.into()),
         content: PostContent::plain(content.unwrap_or_else(|| "".into())),
         urls,
+        time,
         repost_from,
         attachments,
-    }
+    })
 }
 
 enum ReplaceKind<'a> {