assemble: correct handling of fragment length

This feature is unused but has been broken, because we expect users to
provide length limits in MBs but then cast them to bytes.

Fix it to consistently interpret length as MB which are mapped into
bytes when assembling JournalSpecs.

crates/assemble/src/lib.rs

@@ -122,8 +122,8 @@ pub fn partition_template(
     // If an explicit flush interval isn't provided, then don't set one.
     let flush_interval = flush_interval.map(Into::into);
 
-    // If a fragment length isn't set, default to 512MB.
-    let length = length.unwrap_or(1 << 29) as i64;
+    // If a fragment length isn't set, default and then map MB to bytes.
+    let length = (length.unwrap_or(512) as i64) << 20;
 
     // Until there's a good reason otherwise, we hard-code that fragments include the UTC date
     // and hour they were created as components of their path. This makes it easy to filter

crates/validation/tests/model.yaml

@@ -210,7 +210,7 @@ test://example/int-halve:
         Extra: /extra
       journals:
         fragments:
-          length: 11223344
+          length: 100
           flushInterval: 15m
       derive:
         using:

crates/validation/tests/snapshots/scenario_tests__golden_all_visits.snap

@@ -2255,7 +2255,7 @@ Outcome {
               },
               "journals": {
                 "fragments": {
-                  "length": 11223344,
+                  "length": 100,
                   "flushInterval": "15m"
                 }
               },
@@ -2666,7 +2666,7 @@ Outcome {
                         ),
                         fragment: Some(
                             Fragment {
-                                length: 11223344,
+                                length: 104857600,
                                 compression_codec: Gzip,
                                 stores: [
                                     "s3://data-bucket/",
@@ -3477,7 +3477,7 @@ Outcome {
                                                 ),
                                                 fragment: Some(
                                                     Fragment {
-                                                        length: 11223344,
+                                                        length: 104857600,
                                                         compression_codec: Gzip,
                                                         stores: [
                                                             "s3://data-bucket/",
@@ -6726,7 +6726,7 @@ Outcome {
                                         ),
                                         fragment: Some(
                                             Fragment {
-                                                length: 11223344,
+                                                length: 104857600,
                                                 compression_codec: Gzip,
                                                 stores: [
                                                     "s3://data-bucket/",
@@ -7206,7 +7206,7 @@ Outcome {
               },
               "journals": {
                 "fragments": {
-                  "length": 11223344,
+                  "length": 100,
                   "flushInterval": "15m"
                 }
               },
@@ -7652,7 +7652,7 @@ Outcome {
             resource: test://example/int-halve,
             content_type: "CATALOG",
             content: ".. binary ..",
-            content_dom: {"collections":{"testing/int-halve":{"derive":{"transforms":[{"name":"halveIntString","shuffle":{"key":["/len","/str"]},"source":{"name":"testing/int-string-rw","notAfter":"2019-03-06T09:30:02Z","partitions":{"exclude":{"bit":[false]},"include":{"bit":[true]}}}},{"backfill":4,"name":"halveSelf","shuffle":{"key":["/len","/partitionString"]},"source":{"name":"testing/int-halve"}}],"using":{"typescript":{"module":"int-halve.ts"}}},"journals":{"fragments":{"flushInterval":"15m","length":11223344}},"key":["/int"],"projections":{"Extra":"/extra","Len":{"location":"/len","partition":true},"Root":"","TheString":{"location":"/partitionString","partition":true}},"schema":"test://example/int-string-len.schema"}}},
+            content_dom: {"collections":{"testing/int-halve":{"derive":{"transforms":[{"name":"halveIntString","shuffle":{"key":["/len","/str"]},"source":{"name":"testing/int-string-rw","notAfter":"2019-03-06T09:30:02Z","partitions":{"exclude":{"bit":[false]},"include":{"bit":[true]}}}},{"backfill":4,"name":"halveSelf","shuffle":{"key":["/len","/partitionString"]},"source":{"name":"testing/int-halve"}}],"using":{"typescript":{"module":"int-halve.ts"}}},"journals":{"fragments":{"flushInterval":"15m","length":100}},"key":["/int"],"projections":{"Extra":"/extra","Len":{"location":"/len","partition":true},"Root":"","TheString":{"location":"/partitionString","partition":true}},"schema":"test://example/int-string-len.schema"}}},
         },
         Resource {
             resource: test://example/int-halve.ts,