Merge branch 'rc-v0.5.1' into github_commits_graphql

Worklytics · Feb 6, 2025 · 973c1ef · 973c1ef
2 parents cbdd93e + 61e9672
commit 973c1ef
Show file tree

Hide file tree

Showing 10 changed files with 125 additions and 13 deletions.
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/docs/sources/github/example-api-responses/original/org_audit_log.json b/docs/sources/github/example-api-responses/original/org_audit_log.json
@@ -63,5 +63,30 @@
     "user": "octocat",
     "user_agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0",
     "user_id": 13968776
+  },
+  {
+    "repo": [
+      "some-org/test"
+    ],
+    "repo_id": [
+      12324
+    ],
+    "org": "some-org",
+    "org_id": 870,
+    "action": "migration.destroy_file",
+    "@timestamp": 1737730851061,
+    "created_at": 1737730851061,
+    "operation_type": "remove",
+    "data": {
+      "job": "MigrationDestroyFileJob",
+      "active_job_id": "some-job-id",
+      "aqueduct_job_id": "some-aqueduct-job-id",
+      "catalog_service": "github/unknown",
+      "migration_id": 2,
+      "public_repo": true,
+      "started_by": "octocat",
+      "started_by_id": 42,
+      "category_type": "Resource Management"
+    }
   }
-]
+]
diff --git a/docs/sources/github/example-api-responses/sanitized/org_audit_log.json b/docs/sources/github/example-api-responses/sanitized/org_audit_log.json
@@ -12,7 +12,7 @@
     "repository":"some-org/test",
     "repository_public":false,
     "user":"null",
-    "user_id":0
+    "user_id":"{\"hash\":\"uXY-rAXeOzg_TOxA3id5pRgzfCc7sMGRAnsZJK0Ds0U\"}"
   },
   {
     "@timestamp":1688668100244,
@@ -27,7 +27,7 @@
     "repository":"some-org/test",
     "repository_public":false,
     "user":"null",
-    "user_id":0
+    "user_id":"{\"hash\":\"uXY-rAXeOzg_TOxA3id5pRgzfCc7sMGRAnsZJK0Ds0U\"}"
   },
   {
     "@timestamp":1688667848231,
@@ -48,6 +48,26 @@
     "repo":"some-org/test",
     "repository":"some-org/test",
     "user":"{\"hash\":\"IAUEqSLLtP3EjjkzslH-S1ULJZRLQnH9hT54jiI1gbM\"}",
-    "user_id":13968776
+    "user_id":"{\"hash\":\"S3Nh4slc2XnLp5Qy1CD-pQew9-j0cePbX3U7-DE56Co\"}"
+  },
+  {
+    "repo":[
+      "some-org/test"
+    ],
+    "repo_id":[
+      12324
+    ],
+    "org":"some-org",
+    "org_id":870,
+    "action":"migration.destroy_file",
+    "@timestamp":1737730851061,
+    "created_at":1737730851061,
+    "operation_type":"remove",
+    "data":{
+      "public_repo":true,
+      "started_by":"{\"hash\":\"IAUEqSLLtP3EjjkzslH-S1ULJZRLQnH9hT54jiI1gbM\"}",
+      "started_by_id":42,
+      "category_type":"Resource Management"
+    }
   }
 ]
diff --git a/docs/sources/github/github-enterprise-server.yaml b/docs/sources/github/github-enterprise-server.yaml
@@ -464,11 +464,17 @@ endpoints:
           - "$..transport_protocol_name"
           - "$..pull_request_title"
           - "$..user_agent"
+          - "$..job"
+          - "$..active_job_id"
+          - "$..aqueduct_job_id"
+          - "$..catalog_service"
+          - "$..migration_id"
       - !<pseudonymize>
         jsonPaths:
           - "$..actor"
           - "$..user"
-          - "$..userId"
+          - "$..user_id"
+          - "$..started_by"
         encoding: "JSON"
   - pathTemplate: "/api/{enterpriseServerVersion}/organizations/{installationId}/audit-log"
     allowedQueryParams:
@@ -489,11 +495,17 @@ endpoints:
           - "$..transport_protocol_name"
           - "$..pull_request_title"
           - "$..user_agent"
+          - "$..job"
+          - "$..active_job_id"
+          - "$..aqueduct_job_id"
+          - "$..catalog_service"
+          - "$..migration_id"
       - !<pseudonymize>
         jsonPaths:
           - "$..actor"
           - "$..user"
-          - "$..userId"
+          - "$..user_id"
+          - "$..started_by"
         encoding: "JSON"
   - pathTemplate: "/api/{enterpriseServerVersion}/orgs/{org}/repos"
     allowedQueryParams:

diff --git a/docs/sources/github/github.yaml b/docs/sources/github/github.yaml
@@ -464,11 +464,17 @@ endpoints:
           - "$..transport_protocol_name"
           - "$..pull_request_title"
           - "$..user_agent"
+          - "$..job"
+          - "$..active_job_id"
+          - "$..aqueduct_job_id"
+          - "$..catalog_service"
+          - "$..migration_id"
       - !<pseudonymize>
         jsonPaths:
           - "$..actor"
           - "$..user"
-          - "$..userId"
+          - "$..user_id"
+          - "$..started_by"
         encoding: "JSON"
   - pathTemplate: "/organizations/{installationId}/audit-log"
     allowedQueryParams:
@@ -489,11 +495,17 @@ endpoints:
           - "$..transport_protocol_name"
           - "$..pull_request_title"
           - "$..user_agent"
+          - "$..job"
+          - "$..active_job_id"
+          - "$..aqueduct_job_id"
+          - "$..catalog_service"
+          - "$..migration_id"
       - !<pseudonymize>
         jsonPaths:
           - "$..actor"
           - "$..user"
-          - "$..userId"
+          - "$..user_id"
+          - "$..started_by"
         encoding: "JSON"
   - pathTemplate: "/orgs/{org}/repos"
     allowedQueryParams:

diff --git a/java/core/src/main/java/co/worklytics/psoxy/HealthCheckResult.java b/java/core/src/main/java/co/worklytics/psoxy/HealthCheckResult.java
@@ -70,6 +70,14 @@ public void setVersion(String version) {
 
     String callerIp;
 
+    /**
+     *  A SHA-256 hash of the salt, to aid in detecting changes to the salt value.
+     *
+     *  If salt changes, client needs to know; as all subsequent pseudonyms produced by proxy instance from that point
+     *  will be inconsistent with the prior ones.
+     */
+    String saltSha256Hash;
+
     public boolean passed() {
         return getConfiguredSource() != null
             && getNonDefaultSalt()

diff --git a/java/core/src/main/java/co/worklytics/psoxy/gateway/impl/HealthCheckRequestHandler.java b/java/core/src/main/java/co/worklytics/psoxy/gateway/impl/HealthCheckRequestHandler.java
@@ -1,6 +1,7 @@
 package co.worklytics.psoxy.gateway.impl;
 
 import co.worklytics.psoxy.ControlHeader;
+import co.worklytics.psoxy.HashUtils;
 import co.worklytics.psoxy.HealthCheckResult;
 import co.worklytics.psoxy.gateway.*;
 import co.worklytics.psoxy.gateway.impl.oauth.OAuthRefreshTokenSourceAuthStrategy;
@@ -33,6 +34,18 @@ public class HealthCheckRequestHandler {
 
     static final String JAVA_SOURCE_CODE_VERSION = "rc-v0.5.1";
 
+    /**
+     * a random UUID used to salt the hash of the salt.  Purpose of this is to invalidate any non-purpose built rainbow table solution.
+     *   (Eg, if we just directly hashed the salt, a general rainbow table of hashes could be used to determine the salt value)
+     *
+     *  That said, if salt is 20+ random characters, there is no *general* rainbow table of that length in existence and one is impossible to
+     *  build, as storing it requires ~10e25 petabytes - which is about 10e20 more storage than humanity actually has. So this additional
+     *  protection isn't so necessary, but whatever.
+     *
+     *  do NOT change this value. if you do, we won't be able to detect that proxy-side salts of changed.
+     */
+    private static final String SALT_FOR_SALT = "f33c366c-ae91-4819-b221-f9794ebb8145";
+
     @Inject
     EnvVarsConfigService envVarsConfigService;
     @Inject
@@ -45,6 +58,8 @@ public class HealthCheckRequestHandler {
     ObjectMapper objectMapper;
     @Inject
     RulesUtils rulesUtils;
+    @Inject
+    HashUtils hashUtils;
 
     public Optional<HttpEventResponse> handleIfHealthCheck(HttpEventRequest request) {
         if (isHealthCheckRequest(request)) {
@@ -153,6 +168,12 @@ private HttpEventResponse handle(HttpEventRequest request) {
             logInDev("Failed to add rules to health check", e);
         }
 
+        // if SALT configured, as a hash of it to the health check, to enable detection of changes
+        // (if salt changes, client needs to know; as all subsequent pseudonyms produced by proxy instance from that point
+        // will be inconsistent with the prior ones)
+        config.getConfigPropertyAsOptional(ProxyConfigProperty.PSOXY_SALT)
+                .ifPresent(salt -> healthCheckResult.saltSha256Hash(hashUtils.hash(salt, SALT_FOR_SALT)));
+
         HttpEventResponse.HttpEventResponseBuilder responseBuilder = HttpEventResponse.builder();
 
         try {

diff --git a/java/core/src/main/java/co/worklytics/psoxy/rules/github/PrebuiltSanitizerRules.java b/java/core/src/main/java/co/worklytics/psoxy/rules/github/PrebuiltSanitizerRules.java
@@ -246,11 +246,18 @@ public class PrebuiltSanitizerRules {
             .jsonPath("$..transport_protocol_name")
             .jsonPath("$..pull_request_title")
             .jsonPath("$..user_agent")
+            .jsonPath("$..job")
+            .jsonPath("$..active_job_id")
+            .jsonPath("$..aqueduct_job_id")
+            .jsonPath("$..catalog_service")
+            .jsonPath("$..migration_id")
             .build())
         .transform(Transform.Pseudonymize.builder()
             .jsonPath("$..actor")
             .jsonPath("$..user")
             .jsonPath("$..userId")
+            .jsonPath("$..user_id")
+            .jsonPath("$..started_by")
             .build())
         .build();
 
@@ -265,11 +272,18 @@ public class PrebuiltSanitizerRules {
             .jsonPath("$..transport_protocol_name")
             .jsonPath("$..pull_request_title")
             .jsonPath("$..user_agent")
+            .jsonPath("$..job")
+            .jsonPath("$..active_job_id")
+            .jsonPath("$..aqueduct_job_id")
+            .jsonPath("$..catalog_service")
+            .jsonPath("$..migration_id")
             .build())
         .transform(Transform.Pseudonymize.builder()
             .jsonPath("$..actor")
             .jsonPath("$..user")
             .jsonPath("$..userId")
+            .jsonPath("$..user_id")
+            .jsonPath("$..started_by")
             .build())
         .build();
 

diff --git a/java/core/src/test/java/co/worklytics/psoxy/HealthCheckResultTest.java b/java/core/src/test/java/co/worklytics/psoxy/HealthCheckResultTest.java
@@ -30,6 +30,7 @@ public void json() throws JsonProcessingException {
             "  \"nonDefaultSalt\" : true,\n" +
             "  \"pseudonymImplementation\" : null,\n" +
             "  \"pseudonymizeAppIds\" : null,\n" +
+            "  \"saltSha256Hash\" : null,\n" +
             "  \"sourceAuthGrantType\" : null,\n" +
             "  \"sourceAuthStrategy\" : null,\n" +
             "  \"version\" : \"rc-v0.1.15\"\n" +

diff --git a/java/pom.xml b/java/pom.xml
@@ -14,7 +14,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <dependency.lombok.version>1.18.30</dependency.lombok.version> <!-- 1.18.30 is min compatible with jdk 21+ -->
         <dependency.dagger.version>2.40.5</dependency.dagger.version>
-        <dependency.jackson.version>2.15.2</dependency.jackson.version>
+        <dependency.jackson.version>2.17.0</dependency.jackson.version>
         <dependency.apache-commons-lang3.version>3.13.0</dependency.apache-commons-lang3.version> <!-- July 2023 release, doesn't actually have constants for java after 17 yet -->
         <dependency.apache-commons-csv.version>1.10.0</dependency.apache-commons-csv.version>
         <dependency.guava.version>32.0.1-jre</dependency.guava.version>
@@ -25,7 +25,7 @@
         <dependency.google-auth-library-oauth2-http.version>1.18.0</dependency.google-auth-library-oauth2-http.version>
         <dependency.junit-jupiter.version>5.10.1</dependency.junit-jupiter.version>
         <dependency.mockito-junit-jupiter.version>5.14.2</dependency.mockito-junit-jupiter.version>
-        <dependency.json-path.version>2.8.0</dependency.json-path.version>
+        <dependency.json-path.version>2.9.0</dependency.json-path.version>
         <dependency.bettercloud-vault-java-driver>5.1.0</dependency.bettercloud-vault-java-driver>
     </properties>