Skip to content

Commit

Permalink
reorg
Browse files Browse the repository at this point in the history
  • Loading branch information
rorymalcolm committed Jan 26, 2024
1 parent 6a655b7 commit 3234b89
Show file tree
Hide file tree
Showing 3 changed files with 11 additions and 6 deletions.
2 changes: 1 addition & 1 deletion gateway/src/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ const getSchema = async (env: Env): Promise<ValueResult<ParquetSchema>> => {
errors: ['Schema is not valid JSON'],
};
}
const parseResult = ParquetSchema.safeParse(schemaJSON.value);

const parseResult = ParquetSchema.safeParse(schemaJSON.value);
if (parseResult.success) {
return {
success: true,
Expand Down
13 changes: 9 additions & 4 deletions parquet-generator/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ fn physical_type_matcher(parquet_primitive_type: ParquetPrimitiveType) -> Physic
fn build_schema(schema: String) -> String {
let schema = serde_json::from_str::<ParquetSchema>(schema.as_str()).unwrap();
let mut type_vec: Vec<Arc<Type>> = vec![];

for field in schema.fields {
let type_builder = Type::primitive_type_builder(
field.name.as_str(),
Expand All @@ -133,28 +134,32 @@ fn build_schema(schema: String) -> String {
let converted_type = type_builder.build().unwrap();
type_vec.push(Arc::new(converted_type));
}

let mut buf = Vec::new();

let schema = Type::group_type_builder("schema")
.with_fields(type_vec)
.build()
.unwrap();
printer::print_schema(&mut buf, &schema);
let string_schema = String::from_utf8(buf).unwrap();
string_schema

String::from_utf8(buf).unwrap()
}

#[wasm_bindgen]
pub fn generate_parquet(schema: String, fields: Vec<String>) -> Result<(), JsValue> {
let message_type = build_schema(schema);
let schema = Arc::new(parse_message_type(message_type.as_str()).unwrap());

let buffer = vec![];
let schema = Arc::new(parse_message_type(message_type.as_str()).unwrap());

let mut writer = SerializedFileWriter::new(buffer, schema, Default::default()).unwrap();
let mut row_group_writer = writer.next_row_group().unwrap();

while let Some(col_writer) = row_group_writer.next_column().unwrap() {
// ... write values to a column writer
col_writer.close().unwrap()
}

row_group_writer.close().unwrap();
writer.close().unwrap();

Expand Down
2 changes: 1 addition & 1 deletion schema-manager/src/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ async function processGETSchema(env: Env) {
if (!schemaJSON.success) {
return ErrorsToResponse(schemaJSON.errors);
}
const parseResult = ParquetSchema.safeParse(schemaJSON.value);

const parseResult = ParquetSchema.safeParse(schemaJSON.value);
if (parseResult.success) {
return new Response(JSON.stringify({ schema: parseResult.data }));
}
Expand Down

0 comments on commit 3234b89

Please sign in to comment.