Add init feature

README.md

@@ -44,7 +44,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: calvinmclean/article-sync@v1.0.1
+      - uses: calvinmclean/article-sync@v1.2.0
         with:
           type: summary
           api_key: ${{ secrets.DEV_TO_API_KEY }}
@@ -62,7 +62,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: calvinmclean/article-sync@v1.0.1
+      - uses: calvinmclean/article-sync@v1.2.0
         with:
           type: synchronize
           api_key: ${{ secrets.DEV_TO_API_KEY }}
@@ -74,6 +74,15 @@ This works declaratively by parsing each article and:
     - Compare to existing contents fetched by ID
     - Update if changed, otherwise leave alone
 
+## Import Existing Articles
+Simply run the CLI with `--init` flag to initialize a directory structure from existing articles.
+Directory names use the article slug, but can be renamed without affecting the program.
+
+```shell
+go run -mod=mod github.com/calvinmclean/article-sync@latest \
+  --api-key $API_KEY \
+  --init
+```
+
 ## Roadmap
-- Support tags
 - Allow naming files other than `article.md` or `article.json`

main.go

@@ -69,12 +69,13 @@ type commentData struct {
 
 func main() {
 	var apiKey, path, prComment, commit string
-	var dryRun bool
+	var dryRun, init bool
 	flag.StringVar(&apiKey, "api-key", "", "API key for accessing dev.to")
 	flag.StringVar(&path, "path", "./articles", "root path to scan for articles")
 	flag.StringVar(&prComment, "pr-comment", "", "file to write the PR comment into")
 	flag.StringVar(&commit, "commit", "", "file to write the commit message into")
 	flag.BoolVar(&dryRun, "dry-run", false, "dry-run to print which changes will be made without doing them")
+	flag.BoolVar(&init, "init", false, "download articles from profile and create directories")
 	flag.Parse()
 
 	if apiKey == "" {
@@ -89,6 +90,14 @@ func main() {
 		log.Fatalf("error creating API client: %v", err)
 	}
 
+	if init {
+		err = client.init(path)
+		if err != nil {
+			log.Fatalf("error initializing: %v", err)
+		}
+		return
+	}
+
 	var data commentData
 	err = client.syncArticlesFromRootDirectory(path, &data)
 	if err != nil {
@@ -128,6 +137,128 @@ func newClient(apikey string, dryRun bool) (*client, error) {
 	return &client{c, dryRun, slog.New(slog.NewTextHandler(os.Stdout, nil))}, nil
 }
 
+func (c *client) init(path string) error {
+	err := os.MkdirAll(path, 0755)
+	if err != nil {
+		return fmt.Errorf("error creating directory: %w", err)
+	}
+
+	articles, err := c.getPublishedArticles()
+	if err != nil {
+		return fmt.Errorf("error getting articles: %w", err)
+	}
+	c.logger.Info("fetched articles", "count", len(articles))
+
+	existingArticles, err := c.getExistingArticleIDs(path)
+	if err != nil {
+		return fmt.Errorf("error getting existing article IDs: %w", err)
+	}
+	c.logger.Info("existing articles", "count", len(existingArticles))
+
+	for _, a := range articles {
+		logger := c.logger.With("id", a.Id)
+		logger.Info("creating article locally")
+
+		_, exists := existingArticles[int(a.Id)]
+		if exists {
+			logger.Info("article exists")
+			continue
+		}
+
+		fullArticle, err := c.getArticle(int(a.Id))
+		if err != nil {
+			logger.Error("error getting article", "error", err)
+			continue
+		}
+
+		logger.Info("got full article details")
+
+		articleDir := filepath.Join(path, a.Slug)
+		err = os.MkdirAll(articleDir, 0755)
+		if err != nil {
+			return fmt.Errorf("error creating directory: %w", err)
+		}
+
+		logger.Info("created directory", "dir", articleDir)
+
+		err = writeArticleFile(articleDir, &Article{
+			ID:          int(a.Id),
+			Slug:        a.Slug,
+			Title:       a.Title,
+			Description: a.Description,
+			URL:         a.Url,
+			Tags:        a.TagList,
+		})
+		if err != nil {
+			return fmt.Errorf("error writing article JSON file: %w", err)
+		}
+
+		articleMarkdown, ok := fullArticle["body_markdown"].(string)
+		if !ok {
+			return fmt.Errorf("error checking body_markdown")
+		}
+
+		err = os.WriteFile(filepath.Join(articleDir, "article.md"), []byte(articleMarkdown), 0644)
+
+		logger.Info("added files", "dir", articleDir)
+	}
+
+	return nil
+}
+
+func (c *client) getExistingArticleIDs(rootDir string) (map[int]struct{}, error) {
+	result := map[int]struct{}{}
+
+	err := filepath.WalkDir(rootDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return fmt.Errorf("error accessing path %s: %w", path, err)
+		}
+
+		if path == rootDir {
+			return nil
+		}
+
+		if !d.IsDir() {
+			return nil
+		}
+
+		c.logger.Info("checking for article", "directory", path)
+		data, err := os.ReadFile(filepath.Join(path, "article.json"))
+		if err != nil {
+			return fmt.Errorf("error reading JSON file: %w", err)
+		}
+
+		var article *Article
+		err = json.Unmarshal(data, &article)
+		if err != nil {
+			return fmt.Errorf("error parsing article details: %w", err)
+		}
+
+		c.logger.Info("found article", "id", article.ID)
+
+		result[article.ID] = struct{}{}
+
+		return nil
+	})
+
+	return result, err
+}
+
+func (c *client) getPublishedArticles() ([]api.ArticleIndex, error) {
+	resp, err := doWithRetry[*api.GetUserPublishedArticlesResponse](func() (*api.GetUserPublishedArticlesResponse, error) {
+		return c.GetUserPublishedArticlesWithResponse(context.Background(), nil)
+	}, 5, 1*time.Second)
+	if err != nil {
+		return nil, fmt.Errorf("error getting articles: %w", err)
+	}
+
+	if resp.StatusCode() != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status getting articles: %d %s", resp.StatusCode(), string(resp.Body))
+	}
+
+	return *resp.JSON200, nil
+}
+
 func (c *client) syncArticlesFromRootDirectory(rootDir string, data *commentData) error {
 	return filepath.WalkDir(rootDir, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
@@ -230,17 +361,26 @@ func (c *client) syncArticleFromDirectory(dir string) (*Article, error) {
 
 	logger.Info("successfully synchronized article")
 
-	newData, err := json.MarshalIndent(article, "", "    ")
+	err = writeArticleFile(dir, article)
+	if err != nil {
+		return nil, fmt.Errorf("error writing article JSON file: %w", err)
+	}
+
+	return article, nil
+}
+
+func writeArticleFile(path string, article *Article) error {
+	data, err := json.MarshalIndent(article, "", "    ")
 	if err != nil {
-		return nil, fmt.Errorf("error marshaling response JSON to write to file: %w", err)
+		return fmt.Errorf("error marshaling response JSON to write to file: %w", err)
 	}
 
-	err = os.WriteFile(filepath.Join(dir, "article.json"), newData, 0640)
+	err = os.WriteFile(filepath.Join(path, "article.json"), data, 0640)
 	if err != nil {
-		return nil, fmt.Errorf("error writing JSON file: %w", err)
+		return fmt.Errorf("error writing JSON file: %w", err)
 	}
 
-	return article, nil
+	return nil
 }
 
 func (c *client) shouldUpdateArticle(markdownBody string, article *Article) (bool, error) {