From 11d41aeabd3e97a85b0de45c5ad11231dbceadd5 Mon Sep 17 00:00:00 2001 From: Daniel Gregoire Date: Mon, 8 Apr 2024 14:42:41 -0400 Subject: [PATCH 1/6] [pivotal] Simplify for loop, index unused --- pivotal-import/pivotal_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pivotal-import/pivotal_import.py b/pivotal-import/pivotal_import.py index 1399cfe..4ff97d8 100644 --- a/pivotal-import/pivotal_import.py +++ b/pivotal-import/pivotal_import.py @@ -324,7 +324,7 @@ def _get_next_id(): return id def mock_emitter(items): - for ix, item in enumerate(items): + for item in items: entity_id = _get_next_id() created_entity = item["entity"].copy() created_entity["id"] = entity_id From 969881ec46c25ca9a6f8d6ebae91a8b3af1eb2df Mon Sep 17 00:00:00 2001 From: Daniel Gregoire Date: Mon, 8 Apr 2024 14:48:37 -0400 Subject: [PATCH 2/6] [pivotal] Remove unused ids variable --- pivotal-import/pivotal_import.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pivotal-import/pivotal_import.py b/pivotal-import/pivotal_import.py index 4ff97d8..223e904 100644 --- a/pivotal-import/pivotal_import.py +++ b/pivotal-import/pivotal_import.py @@ -45,7 +45,6 @@ def sc_creator(items): """ batch_stories = [] - ids = [] def create_stories(stories): entities = [s["entity"] for s in stories] From 1f3ca256a6bdf8c2340ee50e2c198f23cfff75e7 Mon Sep 17 00:00:00 2001 From: Daniel Gregoire Date: Mon, 8 Apr 2024 17:20:29 -0400 Subject: [PATCH 3/6] [pivotal] Import Pivotal reviews as Shortcut comments In a previous commit, this importer was extended to map all Pivotal story reviewers as Shortcut story followers, so that they will receive in-app notifications about activity on the stories for which they were reviewers in Pivotal. This commit goes a step further and captures the information about reviewer, review type, and review status in a comment on the story. There is explanatory text at the beginning of the comment followed by a Markdown-formatted table for each review which renders nicely in Shortcut. This review-as-comment is created last and is thus the newest comment on the imported story, which means that by default it renders last in the UI (users can choose to order comments in either chronological order). --- pivotal-import/pivotal_import.py | 36 ++++++++- pivotal-import/pivotal_import_test.py | 105 +++++++++++++++++++++----- 2 files changed, 118 insertions(+), 23 deletions(-) diff --git a/pivotal-import/pivotal_import.py b/pivotal-import/pivotal_import.py index 223e904..4c33394 100644 --- a/pivotal-import/pivotal_import.py +++ b/pivotal-import/pivotal_import.py @@ -113,6 +113,8 @@ def parse_priority(priority): "comment": ("comments", parse_comment), "owned by": "owners", "reviewer": "reviewers", + "review type": "review_types", + "review status": "review_states", "task status": "task_states", "task": "task_titles", } @@ -147,6 +149,13 @@ def parse_priority(priority): ], } +review_as_comment_text_prefix = """\\[Pivotal Importer\\] Reviewers have been added as followers on this Shortcut Story. + +The following table describes the state of their reviews when they were imported into Shortcut from Pivotal Tracker: + +| Reviewer | Review Type | Review Status | +|---|---|---|""" + def parse_row(row, headers): d = dict() @@ -203,10 +212,8 @@ def build_entity(ctx, d): if author_id: new_comment["author_id"] = author_id comments.append(new_comment) - if comments: - d["comments"] = comments - elif "comments" in d: - del d["comments"] + # other things we process are reified as comments, + # so we'll add comments to the d later in processing # releases become Shortcut Stories of type "chore" if d["story_type"] == "release": @@ -256,6 +263,19 @@ def build_entity(ctx, d): if reviewer in user_to_sc_id ] + # format table of all reviewers, types, and statuses as a comment on the imported story + if reviewers: + comment_text = review_as_comment_text_prefix + for reviewer, review_type, review_status in zip( + d.get("reviewers", []), + d.get("review_types", []), + d.get("review_states", []), + ): + comment_text += f"\n|{reviewer}|{review_type}|{review_status}|" + comments.append( + {"author_id": d.get("requested_by_id", None), "text": comment_text} + ) + # Custom Fields custom_fields = [] # process priority as Priority custom field @@ -271,6 +291,14 @@ def build_entity(ctx, d): if custom_fields: d["custom_fields"] = custom_fields + # as a last step, ensure comments (both those that were comments + # in Pivotal, and those we add during import to fill feature gaps) + # are all added to the d dict + if comments: + d["comments"] = comments + elif "comments" in d: + del d["comments"] + elif type == "epic": pass diff --git a/pivotal-import/pivotal_import_test.py b/pivotal-import/pivotal_import_test.py index 4e51f1e..3142a7e 100644 --- a/pivotal-import/pivotal_import_test.py +++ b/pivotal-import/pivotal_import_test.py @@ -113,6 +113,92 @@ def test_build_story_with_comments(): } == build_entity(ctx, d) +def test_build_story_with_reviews(): + ctx = create_test_ctx() + rows = [ + # just reviews, no Pivotal comments + { + "story_type": "feature", + "reviewers": ["Emmanuelle Charpentier", "Giorgio Parisi"], + "review_types": ["code", "security"], + "review_states": ["unstarted", "in_review"], + }, + # both Pivotal comments and reviews, which we add as Shortcut comments + { + "story_type": "bug", + "requester": "Daniel McFadden", + "reviewers": ["Emmanuelle Charpentier", "Giorgio Parisi", "Amy Williams"], + "review_types": ["code", "security", "custom qa"], + "review_states": ["unstarted", "in_review", "passed"], + "comments": [ + {"text": "Comment 1"}, + {"text": "Comment 2"}, + {"text": "Comment 3"}, + ], + }, + ] + + assert ( + [ + { + "type": "story", + "entity": { + "story_type": "feature", + "comments": [ + { + "author_id": None, + "text": review_as_comment_text_prefix + + """ +|Emmanuelle Charpentier|code|unstarted| +|Giorgio Parisi|security|in_review|""", + }, + ], + "follower_ids": [ + "emmanuelle_member_id", + "giorgio_member_id", + ], + "labels": [ + {"name": PIVOTAL_TO_SHORTCUT_LABEL}, + {"name": PIVOTAL_TO_SHORTCUT_RUN_LABEL}, + ], + }, + "parsed_row": rows[0], + }, + { + "type": "story", + "entity": { + "story_type": "bug", + "requested_by_id": "daniel_member_id", + "comments": [ + {"text": "Comment 1"}, + {"text": "Comment 2"}, + {"text": "Comment 3"}, + { + "author_id": "daniel_member_id", + "text": review_as_comment_text_prefix + + """ +|Emmanuelle Charpentier|code|unstarted| +|Giorgio Parisi|security|in_review| +|Amy Williams|custom qa|passed|""", + }, + ], + "follower_ids": [ + "emmanuelle_member_id", + "giorgio_member_id", + "amy_member_id", + ], + "labels": [ + {"name": PIVOTAL_TO_SHORTCUT_LABEL}, + {"name": PIVOTAL_TO_SHORTCUT_RUN_LABEL}, + ], + }, + "parsed_row": rows[1], + }, + ] + == [build_entity(ctx, d) for d in rows] + ) + + def test_build_story_priority_mapping(): ctx = create_test_ctx() rows = [ @@ -210,10 +296,6 @@ def test_build_story_user_mapping(): "story_type": "bug", "owners": ["Amy Williams", "Daniel McFadden"], }, - { - "story_type": "chore", - "reviewers": ["Giorgio Parisi", "Emmanuelle Charpentier"], - }, ] assert [ @@ -244,21 +326,6 @@ def test_build_story_user_mapping(): }, "parsed_row": rows[1], }, - { - "type": "story", - "entity": { - "story_type": "chore", - "follower_ids": [ - ctx["user_config"]["Giorgio Parisi"], - ctx["user_config"]["Emmanuelle Charpentier"], - ], - "labels": [ - {"name": PIVOTAL_TO_SHORTCUT_LABEL}, - {"name": PIVOTAL_TO_SHORTCUT_RUN_LABEL}, - ], - }, - "parsed_row": rows[2], - }, ] == [build_entity(ctx, d) for d in rows] From 5f3dd5c17053a26f1075fcb240dc9cc745a24e9f Mon Sep 17 00:00:00 2001 From: Daniel Gregoire Date: Tue, 9 Apr 2024 08:07:25 -0400 Subject: [PATCH 4/6] [pivotal] Ensure pipes in review content don't disturb Markdown Since the reviewer name, review type, and review status are embedded into a Markdown table in a Shortcut comment, we want to ensure that any pipe `|` characters in those string are escaped so that the table renders correctly in Shortcut. --- pivotal-import/pivotal_import.py | 7 +++++++ pivotal-import/pivotal_import_test.py | 11 +++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pivotal-import/pivotal_import.py b/pivotal-import/pivotal_import.py index 4c33394..dfce128 100644 --- a/pivotal-import/pivotal_import.py +++ b/pivotal-import/pivotal_import.py @@ -157,6 +157,10 @@ def parse_priority(priority): |---|---|---|""" +def escape_md_table_syntax(s): + return s.replace("|", "\\|") + + def parse_row(row, headers): d = dict() for ix, val in enumerate(row): @@ -271,6 +275,9 @@ def build_entity(ctx, d): d.get("review_types", []), d.get("review_states", []), ): + reviewer = escape_md_table_syntax(reviewer) + review_type = escape_md_table_syntax(review_type) + review_status = escape_md_table_syntax(review_status) comment_text += f"\n|{reviewer}|{review_type}|{review_status}|" comments.append( {"author_id": d.get("requested_by_id", None), "text": comment_text} diff --git a/pivotal-import/pivotal_import_test.py b/pivotal-import/pivotal_import_test.py index 3142a7e..a0690bf 100644 --- a/pivotal-import/pivotal_import_test.py +++ b/pivotal-import/pivotal_import_test.py @@ -10,6 +10,7 @@ def create_test_ctx(): "Daniel McFadden": "daniel_member_id", "Emmanuelle Charpentier": "emmanuelle_member_id", "Giorgio Parisi": "giorgio_member_id", + "Piper | Barnes": "piper_member_id", }, "workflow_config": {"unstarted": 400001, "started": 400002, "done": 400003}, } @@ -68,14 +69,16 @@ def test_parse_reviewers(): "Amy Williams", "Giorgio Parisi", "Emmanuelle Charpentier", + "Piper | Barnes", ] } == parse_row( [ "Amy Williams", "Giorgio Parisi", "Emmanuelle Charpentier", + "Piper | Barnes", ], - ["reviewer", "reviewer", "reviewer"], + ["reviewer", "reviewer", "reviewer", "reviewer"], ) @@ -127,7 +130,7 @@ def test_build_story_with_reviews(): { "story_type": "bug", "requester": "Daniel McFadden", - "reviewers": ["Emmanuelle Charpentier", "Giorgio Parisi", "Amy Williams"], + "reviewers": ["Emmanuelle Charpentier", "Giorgio Parisi", "Piper | Barnes"], "review_types": ["code", "security", "custom qa"], "review_states": ["unstarted", "in_review", "passed"], "comments": [ @@ -179,13 +182,13 @@ def test_build_story_with_reviews(): + """ |Emmanuelle Charpentier|code|unstarted| |Giorgio Parisi|security|in_review| -|Amy Williams|custom qa|passed|""", +|Piper \\| Barnes|custom qa|passed|""", }, ], "follower_ids": [ "emmanuelle_member_id", "giorgio_member_id", - "amy_member_id", + "piper_member_id", ], "labels": [ {"name": PIVOTAL_TO_SHORTCUT_LABEL}, From e5def775300be99645c4f0b80d9ef74fc7e9979a Mon Sep 17 00:00:00 2001 From: Daniel Gregoire Date: Tue, 9 Apr 2024 08:33:41 -0400 Subject: [PATCH 5/6] [pivotal] Label imported stories that had reviews in Pivotal Since Shortcut does not have a first-class review concept, this commit adds a label of `pivotal-had-review` to every imported story that had at least one review in Pivotal. --- pivotal-import/pivotal_import.py | 4 ++++ pivotal-import/pivotal_import_test.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/pivotal-import/pivotal_import.py b/pivotal-import/pivotal_import.py index dfce128..3b212d7 100644 --- a/pivotal-import/pivotal_import.py +++ b/pivotal-import/pivotal_import.py @@ -31,6 +31,9 @@ """The label associated with all chore stories created from release types in Pivotal.""" PIVOTAL_RELEASE_TYPE_LABEL = "pivotal-release" +"""The label indicating a story had reviews in Pivotal.""" +PIVOTAL_HAD_REVIEW_LABEL = "pivotal-had-review" + def sc_creator(items): """Create Shortcut entities utilizing bulk APIs whenever possible. @@ -266,6 +269,7 @@ def build_entity(ctx, d): for reviewer in reviewers if reviewer in user_to_sc_id ] + d.setdefault("labels", []).append({"name": PIVOTAL_HAD_REVIEW_LABEL}) # format table of all reviewers, types, and statuses as a comment on the imported story if reviewers: diff --git a/pivotal-import/pivotal_import_test.py b/pivotal-import/pivotal_import_test.py index a0690bf..8ea791e 100644 --- a/pivotal-import/pivotal_import_test.py +++ b/pivotal-import/pivotal_import_test.py @@ -163,6 +163,7 @@ def test_build_story_with_reviews(): "labels": [ {"name": PIVOTAL_TO_SHORTCUT_LABEL}, {"name": PIVOTAL_TO_SHORTCUT_RUN_LABEL}, + {"name": PIVOTAL_HAD_REVIEW_LABEL}, ], }, "parsed_row": rows[0], @@ -193,6 +194,7 @@ def test_build_story_with_reviews(): "labels": [ {"name": PIVOTAL_TO_SHORTCUT_LABEL}, {"name": PIVOTAL_TO_SHORTCUT_RUN_LABEL}, + {"name": PIVOTAL_HAD_REVIEW_LABEL}, ], }, "parsed_row": rows[1], From 443c4aae7709f6a79b67f32de4dca018b2abdb1b Mon Sep 17 00:00:00 2001 From: Daniel Gregoire Date: Tue, 9 Apr 2024 08:49:45 -0400 Subject: [PATCH 6/6] [pivotal] Document how Pivotal story reviews are imported --- pivotal-import/README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pivotal-import/README.md b/pivotal-import/README.md index 9162767..dd498b2 100644 --- a/pivotal-import/README.md +++ b/pivotal-import/README.md @@ -16,8 +16,9 @@ In order to run this, you will require a Pivotal account and the ability to sign - Follow instructions printed to the console to ensure the mapping of Pivotal and Shortcut data is complete and correct. - Refer to `data/priorities.csv`, `data/states.csv`, and `data/users.csv` to review these mappings. 1. If the dry-run output looks correct, you can apply the import to your Shortcut workspace by running `make import-apply` -1. Refer to `data/shortcut_imported_entities.csv` to review all epics, stories, etc. imported successfully into Shortcut. - 1. If you find that you need to adjust your configuration or your Pivotal data and try again, you can run `make delete` to review a dry-run and `make delete-apply` to actually delete the imported Shortcut epics and stories listed in that CSV file. + - The console should print a link to an import-specific Shortcut label page that you can review to find all imported Stories and Epics. + - If you run the importer multiple times, you can review all imported Stories and Epics by visiting Settings > Labels and then searching for the `pivotal->shortcut` label and clicking on it. +1. If you find that you need to adjust your configuration or your Pivotal data and try again, you can run `make delete` to review a dry-run and `make delete-apply` to actually delete the imported Shortcut epics and stories listed in `data/shortcut_imported_entities.csv`. You can also archive or delete content in the Shortcut application if needed. # Operation @@ -31,7 +32,10 @@ If `pivotal_import.py` completes without errors, you can run the script with the The following are known limitations: -- **No story reviewers:** Pivotal story reviewers are not imported. +- **Limited story reviews:** Shortcut does not have a feature equivalent to Pivotal story reviews, so they are imported as follows: + - Pivotal story reviewers are imported as Shortcut story followers on the stories they were assigned for review. Shortcut story followers receive updates in their Shortcut Activity Feed for all story updates. + - Imported stories that had Pivotal reviews have an additional comment with a table that lists all of the story reviews from Pivotal (reviewer, review type, and review status). + - Imported stories that had Pivotal reviews have a label in Shortcut of `pivotal-had-review`. - **No story blockers:** Pivotal story blockers (the relationships between stories) are not imported. - **No iterations:** Pivotal iterations are not imported. - **Epics are imported as unstarted:** Imported epics are set to an unstarted "Todo" state.