Skip to content

Commit

Permalink
Everything working.
Browse files Browse the repository at this point in the history
  • Loading branch information
@lriggs
lriggs committed Sep 27, 2023
1 parent e505f45 commit a20e2ec
Show file tree
Hide file tree
Showing 9 changed files with 354 additions and 74 deletions.
15 changes: 13 additions & 2 deletions cpp/src/arrow/buffer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,10 +444,21 @@ class ARROW_EXPORT ResizableBuffer : public MutableBuffer {
return Reserve(sizeof(T) * new_nb_elements);
}

public:
uint8_t* offsetBuffer;
int64_t offsetCapacity;

protected:
ResizableBuffer(uint8_t* data, int64_t size) : MutableBuffer(data, size) {}
ResizableBuffer(uint8_t* data, int64_t size) : MutableBuffer(data, size) {
offsetBuffer = nullptr;
offsetCapacity = 0;

}
ResizableBuffer(uint8_t* data, int64_t size, std::shared_ptr<MemoryManager> mm)
: MutableBuffer(data, size, std::move(mm)) {}
: MutableBuffer(data, size, std::move(mm)) {
offsetBuffer = nullptr;
offsetCapacity = 0;
}
};

/// \defgroup buffer-allocation-functions Functions for allocating buffers
Expand Down
8 changes: 4 additions & 4 deletions cpp/src/gandiva/annotator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ FieldDescriptorPtr Annotator::MakeDesc(FieldPtr field, bool is_output) {
}

if (field->type()->id() == arrow::Type::LIST) {
//std::cout << "LR Annotator::MakeDesc 1" << std::endl;
std::cout << "LR Annotator::MakeDesc 1" << std::endl;
offsets_idx = buffer_count_++;
if (arrow::is_binary_like(field->type()->field(0)->type()->id())) {
child_offsets_idx = buffer_count_++;
Expand Down Expand Up @@ -91,7 +91,7 @@ void Annotator::PrepareBuffersForField(const FieldDescriptor& desc,

if (desc.HasOffsetsIdx()) {
uint8_t* offsets_buf = const_cast<uint8_t*>(array_data.buffers[buffer_idx]->data());
std::cout << "LR Annotator::PrepareBuffersForField setting eval buffer -4 " << &offsets_buf << std::endl;
std::cout << "LR Annotator::PrepareBuffersForField setting eval buffer -4 " << &offsets_buf << " using idx=" << buffer_idx << std::endl;
eval_batch->SetBuffer(desc.offsets_idx(), offsets_buf, array_data.offset);

if (desc.HasChildOffsetsIdx()) {
Expand Down Expand Up @@ -139,7 +139,7 @@ void Annotator::PrepareBuffersForField(const FieldDescriptor& desc,

uint8_t* data_buf =
const_cast<uint8_t*>(array_data.child_data.at(0)->buffers[buffer_idx]->data());
std::cout << "LR Annotator::PrepareBuffersForField setting eval buffer 0 " << &data_buf << std::endl;
std::cout << "LR Annotator::PrepareBuffersForField setting offset eval buffer idx=" << buffer_idx << " data=" << &data_buf << std::endl;
eval_batch->SetBuffer(desc.data_idx(), data_buf, array_data.child_data.at(0)->offset);
//std::cout << "LR Annotator::PrepareBuffersForField 5a" << std::endl;
}
Expand All @@ -158,7 +158,7 @@ void Annotator::PrepareBuffersForField(const FieldDescriptor& desc,
// list data buffer is in child data buffer
uint8_t* data_buf_ptr = reinterpret_cast<uint8_t*>(
array_data.child_data.at(0)->buffers[buffer_idx].get());
std::cout << "LR Annotator::PrepareBuffersForField setting eval buffer 2 " << &data_buf_ptr << std::endl;
std::cout << "LR Annotator::PrepareBuffersForField setting eval data buffer " << buffer_idx << " data=" << &data_buf_ptr << std::endl;

eval_batch->SetBuffer(desc.data_buffer_ptr_idx(), data_buf_ptr,
array_data.child_data.at(0)->offset);
Expand Down
32 changes: 31 additions & 1 deletion cpp/src/gandiva/array_ops.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ int32_t* array_int32_make_array(int64_t context_ptr, int32_t contains_data, int3
//return reinterpret_cast<int32_t*>(ret);
return reinterpret_cast<int32_t*>(ret);
}

/*
int32_t* array_int32_remove(int64_t context_ptr, const int32_t* entry_buf,
int32_t entry_offsets_len, int32_t remove_data, int32_t* out_len) {
//std::cout << "LR array_int32_remove data=" << remove_data
Expand Down Expand Up @@ -125,6 +125,36 @@ int32_t* array_int32_remove(int64_t context_ptr, const int32_t* entry_buf,
//return reinterpret_cast<int32_t*>(ret);
return reinterpret_cast<int32_t*>(ret);
}
*/
int32_t* array_int32_remove(int64_t context_ptr, const int32_t* entry_buf,
int32_t entry_offsets_len, int32_t remove_data, int32_t* out_len) {
//std::cout << "LR array_int32_remove data=" << remove_data
// << " entry_offsets_len " << entry_offsets_len << std::endl;

std::vector<int> newInts;

for (int i = 0; i < entry_offsets_len; i++) {
//std::cout << "LR going to check " << entry_buf + i << std::endl;
int32_t entry_item = *(entry_buf + (i * 1));
//std::cout << "LR checking value " << entry_len << " against target " << remove_data << std::endl;
if (entry_item == remove_data) {
continue;
} else {
newInts.push_back(entry_item);
}
}

*out_len = newInts.size();
int32_t outBufferLength = *out_len * sizeof(int);
//length is number of items, but buffers must account for byte size.
uint8_t* ret = gdv_fn_context_arena_malloc(context_ptr, outBufferLength);
memcpy(ret, newInts.data(), outBufferLength);
//std::cout << "LR made a buffer length" << *out_len * 4 << " item 3 is = " << int32_t(ret[3*4]) << std::endl;


//return reinterpret_cast<int32_t*>(ret);
return reinterpret_cast<int32_t*>(ret);
}

int64_t array_utf8_length(int64_t context_ptr, const char* entry_buf,
int32_t* entry_child_offsets, int32_t entry_offsets_len) {
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/gandiva/decimal_ir.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ namespace gandiva {
class DecimalIR : public FunctionIRBuilder {
public:
explicit DecimalIR(Engine* engine)
: FunctionIRBuilder(engine), enable_ir_traces_(true) {}
: FunctionIRBuilder(engine), enable_ir_traces_(false) {}

/// Build decimal IR functions and add them to the engine.
static Status AddFunctions(Engine* engine);
Expand Down
10 changes: 8 additions & 2 deletions cpp/src/gandiva/gdv_function_stubs.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,14 +173,20 @@ int32_t gdv_fn_populate_varlen_vector(int64_t context_ptr, int8_t* data_ptr,
context->set_error_msg(status.message().c_str()); \
return -1; \
} \
std::cout << "LR populate_list slot " << slot << " offset = " << offset << " buffer = " << \
(int64_t)(buffer->mutable_data() + offset) << std::endl; \
memcpy(buffer->mutable_data() + offset, (char*)entry_buf, entry_len * SCALE); \
std::cout << "LR gdv_fn_populate buffer=" << buffer->data() << std::endl; \
std::cout << " and offset=" << offsets << " * =" << *offsets << std::endl; \
std::cout << "Setting offset slot=" << slot << "=" << offset / SCALE << std::endl; \
std::cout << "Setting offset slot+1=" << slot + 1 << "=" << offset / SCALE + entry_len << std::endl; \
offsets = reinterpret_cast<int32_t*>(buffer->offsetBuffer); \
offsets[slot] = offset / SCALE; \
offsets[slot + 1] = offset / SCALE + entry_len; \
return 0; \
}

//buffer->offsetBuffer[slot] = offset / SCALE;
//buffer->offsetBuffer[slot + 1] = offset / SCALE + entry_len;

POPULATE_NUMERIC_LIST_TYPE_VECTOR(int32_t, 4)
POPULATE_NUMERIC_LIST_TYPE_VECTOR(int64_t, 8)
POPULATE_NUMERIC_LIST_TYPE_VECTOR(float, 4)
Expand Down
5 changes: 3 additions & 2 deletions cpp/src/gandiva/llvm_generator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ namespace gandiva {
}
}*/

LLVMGenerator::LLVMGenerator(bool cached) : cached_(cached), enable_ir_traces_(true) {}
LLVMGenerator::LLVMGenerator(bool cached) : cached_(cached), enable_ir_traces_(false) {}

Status LLVMGenerator::Make(std::shared_ptr<Configuration> config, bool cached,
std::unique_ptr<LLVMGenerator>* llvm_generator) {
Expand Down Expand Up @@ -472,7 +472,8 @@ Status LLVMGenerator::CodeGenExprValue(DexPtr value_expr, int buffer_count,
output_value->length()->print(output2);


//std::cout << "LR gdv_fn_populate_list_int32_t_vector params are " << arg_context_ptr << "," << output_buffer_ptr_ref << ","
std::cout << "LR gdv_fn_populate_list_int32_t_vector params are " << arg_context_ptr << "," << output_buffer_ptr_ref << ","
<< output_offset_ref << "," << loop_var << std::endl;
// << output_offset_ref << "," << loop_var << "[[" << str1 << "]] [[" << str2 << "]]" << std::endl;
AddFunctionCall("gdv_fn_populate_list_int32_t_vector", types()->i32_type(),
{arg_context_ptr, output_buffer_ptr_ref, output_offset_ref,
Expand Down
Loading

0 comments on commit a20e2ec

Please sign in to comment.