diff --git a/libs/utils/close.go b/libs/utils/close.go index cbcc7b6f67..c9a2ddaa30 100644 --- a/libs/utils/close.go +++ b/libs/utils/close.go @@ -6,6 +6,8 @@ import ( logging "github.com/ipfs/go-log/v2" ) +// CloseAndLog closes the closer and logs any error that occurs. The function is handy wrapping +// to group closing and logging in one call for defer statements. func CloseAndLog(log logging.StandardLogger, name string, closer io.Closer) { if err := closer.Close(); err != nil { log.Warnf("closing %s: %s", name, err) diff --git a/share/shwap/namespace_data.go b/share/shwap/namespace_data.go index 62b157e783..5e41016a1f 100644 --- a/share/shwap/namespace_data.go +++ b/share/shwap/namespace_data.go @@ -40,20 +40,35 @@ func (nd NamespacedData) Validate(root *share.AxisRoots, namespace share.Namespa } // ReadFrom reads the binary form of NamespacedData from the provided reader. -func (nd *NamespacedData) ReadFrom(reader io.Reader) error { +func (nd *NamespacedData) ReadFrom(reader io.Reader) (int64, error) { var data []RowNamespaceData + var n int64 for { var pbRow pb.RowNamespaceData - _, err := serde.Read(reader, &pbRow) + nn, err := serde.Read(reader, &pbRow) + n += int64(nn) if err != nil { if errors.Is(err, io.EOF) { // all rows have been read *nd = data - return nil + return n, nil } - return err + return n, err } row := RowNamespaceDataFromProto(&pbRow) data = append(data, row) } } + +func (nd NamespacedData) WriteTo(writer io.Writer) (int64, error) { + var n int64 + for _, row := range nd { + pbRow := row.ToProto() + nn, err := serde.Write(writer, pbRow) + n += int64(nn) + if err != nil { + return n, err + } + } + return n, nil +} diff --git a/share/shwap/p2p/shrex/shrexeds/pb/extended_data_square.pb.go b/share/shwap/p2p/shrex/pb/shrex.pb.go similarity index 52% rename from share/shwap/p2p/shrex/shrexeds/pb/extended_data_square.pb.go rename to share/shwap/p2p/shrex/pb/shrex.pb.go index 60d26abdc2..88322b1985 100644 --- a/share/shwap/p2p/shrex/shrexeds/pb/extended_data_square.pb.go +++ b/share/shwap/p2p/shrex/pb/shrex.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: share/shwap/p2p/shrex/shrexeds/pb/extended_data_square.proto +// source: share/shwap/p2p/shrex/pb/shrex.proto package pb @@ -50,25 +50,25 @@ func (x Status) String() string { } func (Status) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_5176f06f10cac3fd, []int{0} + return fileDescriptor_68dfd78ade756110, []int{0} } -type EDSResponse struct { +type Response struct { Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=Status" json:"status,omitempty"` } -func (m *EDSResponse) Reset() { *m = EDSResponse{} } -func (m *EDSResponse) String() string { return proto.CompactTextString(m) } -func (*EDSResponse) ProtoMessage() {} -func (*EDSResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_5176f06f10cac3fd, []int{0} +func (m *Response) Reset() { *m = Response{} } +func (m *Response) String() string { return proto.CompactTextString(m) } +func (*Response) ProtoMessage() {} +func (*Response) Descriptor() ([]byte, []int) { + return fileDescriptor_68dfd78ade756110, []int{0} } -func (m *EDSResponse) XXX_Unmarshal(b []byte) error { +func (m *Response) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *EDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_EDSResponse.Marshal(b, m, deterministic) + return xxx_messageInfo_Response.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -78,19 +78,19 @@ func (m *EDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) return b[:n], nil } } -func (m *EDSResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_EDSResponse.Merge(m, src) +func (m *Response) XXX_Merge(src proto.Message) { + xxx_messageInfo_Response.Merge(m, src) } -func (m *EDSResponse) XXX_Size() int { +func (m *Response) XXX_Size() int { return m.Size() } -func (m *EDSResponse) XXX_DiscardUnknown() { - xxx_messageInfo_EDSResponse.DiscardUnknown(m) +func (m *Response) XXX_DiscardUnknown() { + xxx_messageInfo_Response.DiscardUnknown(m) } -var xxx_messageInfo_EDSResponse proto.InternalMessageInfo +var xxx_messageInfo_Response proto.InternalMessageInfo -func (m *EDSResponse) GetStatus() Status { +func (m *Response) GetStatus() Status { if m != nil { return m.Status } @@ -99,34 +99,32 @@ func (m *EDSResponse) GetStatus() Status { func init() { proto.RegisterEnum("Status", Status_name, Status_value) - proto.RegisterType((*EDSResponse)(nil), "EDSResponse") + proto.RegisterType((*Response)(nil), "Response") } func init() { - proto.RegisterFile("share/shwap/p2p/shrex/shrexeds/pb/extended_data_square.proto", fileDescriptor_5176f06f10cac3fd) + proto.RegisterFile("share/shwap/p2p/shrex/pb/shrex.proto", fileDescriptor_68dfd78ade756110) } -var fileDescriptor_5176f06f10cac3fd = []byte{ - // 244 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xb2, 0x29, 0xce, 0x48, 0x2c, +var fileDescriptor_68dfd78ade756110 = []byte{ + // 218 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x29, 0xce, 0x48, 0x2c, 0x4a, 0xd5, 0x2f, 0xce, 0x28, 0x4f, 0x2c, 0xd0, 0x2f, 0x30, 0x2a, 0xd0, 0x2f, 0xce, 0x28, 0x4a, - 0xad, 0x80, 0x90, 0xa9, 0x29, 0xc5, 0xfa, 0x05, 0x49, 0xfa, 0xa9, 0x15, 0x25, 0xa9, 0x79, 0x29, - 0xa9, 0x29, 0xf1, 0x29, 0x89, 0x25, 0x89, 0xf1, 0xc5, 0x85, 0xa5, 0x89, 0x45, 0xa9, 0x7a, 0x05, - 0x45, 0xf9, 0x25, 0xf9, 0x4a, 0x7a, 0x5c, 0xdc, 0xae, 0x2e, 0xc1, 0x41, 0xa9, 0xc5, 0x05, 0xf9, - 0x79, 0xc5, 0xa9, 0x42, 0xf2, 0x5c, 0x6c, 0xc5, 0x25, 0x89, 0x25, 0xa5, 0xc5, 0x12, 0x8c, 0x0a, - 0x8c, 0x1a, 0x7c, 0x46, 0xec, 0x7a, 0xc1, 0x60, 0x6e, 0x10, 0x54, 0x58, 0xcb, 0x8a, 0x8b, 0x0d, - 0x22, 0x22, 0xc4, 0xcd, 0xc5, 0xee, 0xe9, 0x17, 0xe6, 0xe8, 0xe3, 0xe9, 0x22, 0xc0, 0x20, 0xc4, - 0xc6, 0xc5, 0xe4, 0xef, 0x2d, 0xc0, 0x28, 0xc4, 0xcb, 0xc5, 0xe9, 0xe7, 0x1f, 0x12, 0xef, 0xe6, - 0x1f, 0xea, 0xe7, 0x22, 0xc0, 0x24, 0xc4, 0xc3, 0xc5, 0xe1, 0xe9, 0x17, 0xe2, 0x1a, 0xe4, 0xe7, - 0xe8, 0x23, 0xc0, 0xec, 0x94, 0x70, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, - 0xc9, 0x31, 0x4e, 0x78, 0x2c, 0xc7, 0x70, 0xe1, 0xb1, 0x1c, 0xc3, 0x8d, 0xc7, 0x72, 0x0c, 0x51, - 0x6e, 0xe9, 0x99, 0x25, 0x19, 0xa5, 0x49, 0x7a, 0xc9, 0xf9, 0xb9, 0xfa, 0xc9, 0xa9, 0x39, 0xa9, - 0xc5, 0x25, 0x99, 0x89, 0xf9, 0x45, 0xe9, 0x70, 0xb6, 0x6e, 0x5e, 0x7e, 0x0a, 0xc8, 0x8b, 0x04, - 0x3c, 0x9a, 0xc4, 0x06, 0xf6, 0x94, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x91, 0x62, 0x7c, 0xa1, - 0x14, 0x01, 0x00, 0x00, + 0xad, 0xd0, 0x2f, 0x48, 0x82, 0x30, 0xf4, 0x0a, 0x8a, 0xf2, 0x4b, 0xf2, 0x95, 0xb4, 0xb9, 0x38, + 0x82, 0x52, 0x8b, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0xe4, 0xb9, 0xd8, 0x8a, 0x4b, 0x12, 0x4b, + 0x4a, 0x8b, 0x25, 0x18, 0x15, 0x18, 0x35, 0xf8, 0x8c, 0xd8, 0xf5, 0x82, 0xc1, 0xdc, 0x20, 0xa8, + 0xb0, 0x96, 0x15, 0x17, 0x1b, 0x44, 0x44, 0x88, 0x9b, 0x8b, 0xdd, 0xd3, 0x2f, 0xcc, 0xd1, 0xc7, + 0xd3, 0x45, 0x80, 0x41, 0x88, 0x8d, 0x8b, 0xc9, 0xdf, 0x5b, 0x80, 0x51, 0x88, 0x97, 0x8b, 0xd3, + 0xcf, 0x3f, 0x24, 0xde, 0xcd, 0x3f, 0xd4, 0xcf, 0x45, 0x80, 0x49, 0x88, 0x87, 0x8b, 0xc3, 0xd3, + 0x2f, 0xc4, 0x35, 0xc8, 0xcf, 0xd1, 0x47, 0x80, 0xd9, 0x29, 0xfc, 0xc4, 0x23, 0x39, 0xc6, 0x0b, + 0x8f, 0xe4, 0x18, 0x1f, 0x3c, 0x92, 0x63, 0x9c, 0xf0, 0x58, 0x8e, 0xe1, 0xc2, 0x63, 0x39, 0x86, + 0x1b, 0x8f, 0xe5, 0x18, 0xa2, 0x6c, 0xd3, 0x33, 0x4b, 0x32, 0x4a, 0x93, 0xf4, 0x92, 0xf3, 0x73, + 0xf5, 0x93, 0x53, 0x73, 0x52, 0x8b, 0x4b, 0x32, 0x13, 0xf3, 0x8b, 0xd2, 0xe1, 0x6c, 0xdd, 0xbc, + 0xfc, 0x14, 0x90, 0x3f, 0xb0, 0xfb, 0x26, 0x89, 0x0d, 0xec, 0x11, 0x63, 0x40, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xd8, 0xb0, 0x4b, 0x58, 0xf0, 0x00, 0x00, 0x00, } -func (m *EDSResponse) Marshal() (dAtA []byte, err error) { +func (m *Response) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -136,26 +134,26 @@ func (m *EDSResponse) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *EDSResponse) MarshalTo(dAtA []byte) (int, error) { +func (m *Response) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *EDSResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *Response) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l if m.Status != 0 { - i = encodeVarintExtendedDataSquare(dAtA, i, uint64(m.Status)) + i = encodeVarintShrex(dAtA, i, uint64(m.Status)) i-- dAtA[i] = 0x8 } return len(dAtA) - i, nil } -func encodeVarintExtendedDataSquare(dAtA []byte, offset int, v uint64) int { - offset -= sovExtendedDataSquare(v) +func encodeVarintShrex(dAtA []byte, offset int, v uint64) int { + offset -= sovShrex(v) base := offset for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -165,25 +163,25 @@ func encodeVarintExtendedDataSquare(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } -func (m *EDSResponse) Size() (n int) { +func (m *Response) Size() (n int) { if m == nil { return 0 } var l int _ = l if m.Status != 0 { - n += 1 + sovExtendedDataSquare(uint64(m.Status)) + n += 1 + sovShrex(uint64(m.Status)) } return n } -func sovExtendedDataSquare(x uint64) (n int) { +func sovShrex(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } -func sozExtendedDataSquare(x uint64) (n int) { - return sovExtendedDataSquare(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +func sozShrex(x uint64) (n int) { + return sovShrex(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *EDSResponse) Unmarshal(dAtA []byte) error { +func (m *Response) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -191,7 +189,7 @@ func (m *EDSResponse) Unmarshal(dAtA []byte) error { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowExtendedDataSquare + return ErrIntOverflowShrex } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -206,10 +204,10 @@ func (m *EDSResponse) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: EDSResponse: wiretype end group for non-group") + return fmt.Errorf("proto: Response: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: EDSResponse: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Response: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -219,7 +217,7 @@ func (m *EDSResponse) Unmarshal(dAtA []byte) error { m.Status = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowExtendedDataSquare + return ErrIntOverflowShrex } if iNdEx >= l { return io.ErrUnexpectedEOF @@ -233,12 +231,12 @@ func (m *EDSResponse) Unmarshal(dAtA []byte) error { } default: iNdEx = preIndex - skippy, err := skipExtendedDataSquare(dAtA[iNdEx:]) + skippy, err := skipShrex(dAtA[iNdEx:]) if err != nil { return err } if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthExtendedDataSquare + return ErrInvalidLengthShrex } if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF @@ -252,7 +250,7 @@ func (m *EDSResponse) Unmarshal(dAtA []byte) error { } return nil } -func skipExtendedDataSquare(dAtA []byte) (n int, err error) { +func skipShrex(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 depth := 0 @@ -260,7 +258,7 @@ func skipExtendedDataSquare(dAtA []byte) (n int, err error) { var wire uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowExtendedDataSquare + return 0, ErrIntOverflowShrex } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -277,7 +275,7 @@ func skipExtendedDataSquare(dAtA []byte) (n int, err error) { case 0: for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowExtendedDataSquare + return 0, ErrIntOverflowShrex } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -293,7 +291,7 @@ func skipExtendedDataSquare(dAtA []byte) (n int, err error) { var length int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return 0, ErrIntOverflowExtendedDataSquare + return 0, ErrIntOverflowShrex } if iNdEx >= l { return 0, io.ErrUnexpectedEOF @@ -306,14 +304,14 @@ func skipExtendedDataSquare(dAtA []byte) (n int, err error) { } } if length < 0 { - return 0, ErrInvalidLengthExtendedDataSquare + return 0, ErrInvalidLengthShrex } iNdEx += length case 3: depth++ case 4: if depth == 0 { - return 0, ErrUnexpectedEndOfGroupExtendedDataSquare + return 0, ErrUnexpectedEndOfGroupShrex } depth-- case 5: @@ -322,7 +320,7 @@ func skipExtendedDataSquare(dAtA []byte) (n int, err error) { return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } if iNdEx < 0 { - return 0, ErrInvalidLengthExtendedDataSquare + return 0, ErrInvalidLengthShrex } if depth == 0 { return iNdEx, nil @@ -332,7 +330,7 @@ func skipExtendedDataSquare(dAtA []byte) (n int, err error) { } var ( - ErrInvalidLengthExtendedDataSquare = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowExtendedDataSquare = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupExtendedDataSquare = fmt.Errorf("proto: unexpected end of group") + ErrInvalidLengthShrex = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowShrex = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupShrex = fmt.Errorf("proto: unexpected end of group") ) diff --git a/share/shwap/p2p/shrex/shrexeds/pb/extended_data_square.proto b/share/shwap/p2p/shrex/pb/shrex.proto similarity index 82% rename from share/shwap/p2p/shrex/shrexeds/pb/extended_data_square.proto rename to share/shwap/p2p/shrex/pb/shrex.proto index d8d2cf6a52..7b3a57227d 100644 --- a/share/shwap/p2p/shrex/shrexeds/pb/extended_data_square.proto +++ b/share/shwap/p2p/shrex/pb/shrex.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -option go_package = "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexeds/pb"; +option go_package = "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/pb"; enum Status { INVALID = 0; @@ -9,6 +9,6 @@ enum Status { INTERNAL = 3; // internal server error } -message EDSResponse { +message Response { Status status = 1; } diff --git a/share/shwap/p2p/shrex/shrex_getter/shrex_test.go b/share/shwap/p2p/shrex/shrex_getter/shrex_test.go index cb4a5a10ed..5f1e1169a0 100644 --- a/share/shwap/p2p/shrex/shrex_getter/shrex_test.go +++ b/share/shwap/p2p/shrex/shrex_getter/shrex_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/binary" "errors" + "sync/atomic" "testing" "time" @@ -14,7 +15,6 @@ import ( mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/atomic" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/nmt" @@ -62,7 +62,9 @@ func TestShrexGetter(t *testing.T) { getter := NewGetter(edsClient, ndClient, fullPeerManager, archivalPeerManager, light.Window) require.NoError(t, getter.Start(ctx)) - height := atomic.NewUint64(1) + height := atomic.Uint64{} + height.Add(1) + t.Run("ND_Available, total data size > 1mb", func(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second*10) t.Cleanup(cancel) diff --git a/share/shwap/p2p/shrex/shrexeds/client.go b/share/shwap/p2p/shrex/shrexeds/client.go index 7dad597543..5aa5005d94 100644 --- a/share/shwap/p2p/shrex/shrexeds/client.go +++ b/share/shwap/p2p/shrex/shrexeds/client.go @@ -21,7 +21,7 @@ import ( eds "github.com/celestiaorg/celestia-node/share/new_eds" "github.com/celestiaorg/celestia-node/share/shwap" "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" - pb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexeds/pb" + shrexpb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/pb" ) // Client is responsible for requesting EDSs for blocksync over the ShrEx/EDS protocol. @@ -99,32 +99,27 @@ func (c *Client) doRequest( defer stream.Close() c.setStreamDeadlines(ctx, stream) - - req, err := shwap.NewEdsID(height) - if err != nil { - return nil, fmt.Errorf("create request: %w", err) - } - rb, err := req.MarshalBinary() - if err != nil { - return nil, fmt.Errorf("marshal request: %w", err) - } - // request ODS log.Debugw("client: requesting ods", "height", height, "peer", to.String()) - _, err = stream.Write(rb) + id, err := shwap.NewEdsID(height) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + _, err = id.WriteTo(stream) if err != nil { stream.Reset() //nolint:errcheck return nil, fmt.Errorf("write request to stream: %w", err) } + err = stream.CloseWrite() if err != nil { log.Debugw("client: error closing write", "err", err) } // read and parse status from peer - resp := new(pb.EDSResponse) + resp := new(shrexpb.Response) err = stream.SetReadDeadline(time.Now().Add(c.params.ServerReadTimeout)) if err != nil { log.Debugw("client: failed to set read deadline for reading status", "err", err) @@ -140,7 +135,7 @@ func (c *Client) doRequest( return nil, fmt.Errorf("read status from stream: %w", err) } switch resp.Status { - case pb.Status_OK: + case shrexpb.Status_OK: // reset stream deadlines to original values, since read deadline was changed during status read c.setStreamDeadlines(ctx, stream) // use header and ODS bytes to construct EDS and verify it against dataHash @@ -150,13 +145,13 @@ func (c *Client) doRequest( } c.metrics.ObserveRequests(ctx, 1, shrex.StatusSuccess) return eds, nil - case pb.Status_NOT_FOUND: + case shrexpb.Status_NOT_FOUND: c.metrics.ObserveRequests(ctx, 1, shrex.StatusNotFound) return nil, shrex.ErrNotFound - case pb.Status_INVALID: + case shrexpb.Status_INVALID: log.Debug("client: invalid request") fallthrough - case pb.Status_INTERNAL: + case shrexpb.Status_INTERNAL: fallthrough default: c.metrics.ObserveRequests(ctx, 1, shrex.StatusInternalErr) diff --git a/share/shwap/p2p/shrex/shrexeds/params.go b/share/shwap/p2p/shrex/shrexeds/params.go index 4c3667033d..7505d0daf7 100644 --- a/share/shwap/p2p/shrex/shrexeds/params.go +++ b/share/shwap/p2p/shrex/shrexeds/params.go @@ -8,7 +8,7 @@ import ( "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" ) -const protocolString = "/shrex/eds/v0.0.2" +const protocolString = "/shrex/eds/v0.1.0" var log = logging.Logger("shrex/eds") diff --git a/share/shwap/p2p/shrex/shrexeds/server.go b/share/shwap/p2p/shrex/shrexeds/server.go index dbcfcadc4a..6d0cb27f0f 100644 --- a/share/shwap/p2p/shrex/shrexeds/server.go +++ b/share/shwap/p2p/shrex/shrexeds/server.go @@ -18,7 +18,7 @@ import ( eds "github.com/celestiaorg/celestia-node/share/new_eds" "github.com/celestiaorg/celestia-node/share/shwap" "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" - p2p_pb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexeds/pb" + shrexpb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/pb" "github.com/celestiaorg/celestia-node/store" ) @@ -94,18 +94,18 @@ func (s *Server) handleStream(stream network.Stream) { // we do not close the reader, so that other requests will not need to re-open the file. // closing is handled by the LRU cache. file, err := s.store.GetByHeight(ctx, id.Height) - var status p2p_pb.Status + var status shrexpb.Status switch { case err == nil: defer utils.CloseAndLog(logger, "file", file) - status = p2p_pb.Status_OK + status = shrexpb.Status_OK case errors.Is(err, store.ErrNotFound): logger.Warnw("server: request height not found") s.metrics.ObserveRequests(ctx, 1, shrex.StatusNotFound) - status = p2p_pb.Status_NOT_FOUND + status = shrexpb.Status_NOT_FOUND case err != nil: logger.Errorw("server: get file", "err", err) - status = p2p_pb.Status_INTERNAL + status = shrexpb.Status_INTERNAL } // inform the client of our status @@ -116,7 +116,7 @@ func (s *Server) handleStream(stream network.Stream) { return } // if we cannot serve the EDS, we are already done - if status != p2p_pb.Status_OK { + if status != shrexpb.Status_OK { err = stream.Close() if err != nil { logger.Debugw("server: closing stream", "err", err) @@ -145,36 +145,31 @@ func (s *Server) readRequest(logger *zap.SugaredLogger, stream network.Stream) ( logger.Debugw("server: set read deadline", "err", err) } - req := make([]byte, shwap.EdsIDSize) - _, err = io.ReadFull(stream, req) + edsIds := shwap.EdsID{} + _, err = edsIds.ReadFrom(stream) if err != nil { return shwap.EdsID{}, fmt.Errorf("reading request: %w", err) } - id, err := shwap.EdsIDFromBinary(req) - if err != nil { - return shwap.EdsID{}, fmt.Errorf("parsing request: %w", err) - } err = stream.CloseRead() if err != nil { logger.Debugw("server: closing read", "err", err) } - - return id, id.Validate() + return edsIds, nil } -func (s *Server) writeStatus(logger *zap.SugaredLogger, status p2p_pb.Status, stream network.Stream) error { +func (s *Server) writeStatus(logger *zap.SugaredLogger, status shrexpb.Status, stream network.Stream) error { err := stream.SetWriteDeadline(time.Now().Add(s.params.ServerWriteTimeout)) if err != nil { logger.Debugw("server: set write deadline", "err", err) } - resp := &p2p_pb.EDSResponse{Status: status} + resp := &shrexpb.Response{Status: status} _, err = serde.Write(stream, resp) return err } -func (s *Server) writeODS(logger *zap.SugaredLogger, stramer eds.Streamer, stream network.Stream) error { - reader, err := stramer.Reader() +func (s *Server) writeODS(logger *zap.SugaredLogger, streamer eds.Streamer, stream network.Stream) error { + reader, err := streamer.Reader() if err != nil { return fmt.Errorf("getting ODS reader: %w", err) } diff --git a/share/shwap/p2p/shrex/shrexnd/client.go b/share/shwap/p2p/shrex/shrexnd/client.go index b8bf6db8e6..02d6f0d345 100644 --- a/share/shwap/p2p/shrex/shrexnd/client.go +++ b/share/shwap/p2p/shrex/shrexnd/client.go @@ -14,12 +14,11 @@ import ( "github.com/libp2p/go-libp2p/core/protocol" "github.com/celestiaorg/go-libp2p-messenger/serde" - "github.com/celestiaorg/nmt" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/shwap" "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" - pb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexnd/pb" + shrexpb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/pb" ) // Client implements client side of shrex/nd protocol to obtain namespaced shares data from remote @@ -99,12 +98,7 @@ func (c *Client) doRequest( return nil, fmt.Errorf("client-nd: creating request: %w", err) } - br, err := req.MarshalBinary() - if err != nil { - return nil, fmt.Errorf("client-nd: marshaling request: %w", err) - } - - _, err = stream.Write(br) + _, err = req.WriteTo(stream) if err != nil { c.metrics.ObserveRequests(ctx, 1, shrex.StatusSendReqErr) stream.Reset() //nolint:errcheck @@ -119,11 +113,18 @@ func (c *Client) doRequest( if err := c.readStatus(ctx, stream); err != nil { return nil, err } - return c.readNamespacedShares(ctx, stream) + + nd := shwap.NamespacedData{} + _, err = nd.ReadFrom(stream) + if err != nil { + c.metrics.ObserveRequests(ctx, 1, shrex.StatusReadRespErr) + return nil, err + } + return nd, nil } func (c *Client) readStatus(ctx context.Context, stream network.Stream) error { - var resp pb.GetSharesByNamespaceStatusResponse + var resp shrexpb.Response _, err := serde.Read(stream, &resp) if err != nil { // server is overloaded and closed the stream @@ -139,49 +140,6 @@ func (c *Client) readStatus(ctx context.Context, stream network.Stream) error { return c.convertStatusToErr(ctx, resp.Status) } -// readNamespacedShares converts proto Rows to share.NamespacedData -func (c *Client) readNamespacedShares( - ctx context.Context, - stream network.Stream, -) (shwap.NamespacedData, error) { - var shares shwap.NamespacedData - for { - var row pb.NamespaceRowResponse - _, err := serde.Read(stream, &row) - if err != nil { - if errors.Is(err, io.EOF) { - // all data is received and steam is closed by server - return shares, nil - } - c.metrics.ObserveRequests(ctx, 1, shrex.StatusReadRespErr) - return nil, err - } - var proof nmt.Proof - if row.Proof != nil { - if len(row.Shares) != 0 { - proof = nmt.NewInclusionProof( - int(row.Proof.Start), - int(row.Proof.End), - row.Proof.Nodes, - row.Proof.IsMaxNamespaceIgnored, - ) - } else { - proof = nmt.NewAbsenceProof( - int(row.Proof.Start), - int(row.Proof.End), - row.Proof.Nodes, - row.Proof.LeafHash, - row.Proof.IsMaxNamespaceIgnored, - ) - } - } - shares = append(shares, shwap.RowNamespaceData{ - Shares: row.Shares, - Proof: &proof, - }) - } -} - func (c *Client) setStreamDeadlines(ctx context.Context, stream network.Stream) { // set read/write deadline to use context deadline if it exists deadline, ok := ctx.Deadline() @@ -210,18 +168,18 @@ func (c *Client) setStreamDeadlines(ctx context.Context, stream network.Stream) } } -func (c *Client) convertStatusToErr(ctx context.Context, status pb.StatusCode) error { +func (c *Client) convertStatusToErr(ctx context.Context, status shrexpb.Status) error { switch status { - case pb.StatusCode_OK: + case shrexpb.Status_OK: c.metrics.ObserveRequests(ctx, 1, shrex.StatusSuccess) return nil - case pb.StatusCode_NOT_FOUND: + case shrexpb.Status_NOT_FOUND: c.metrics.ObserveRequests(ctx, 1, shrex.StatusNotFound) return shrex.ErrNotFound - case pb.StatusCode_INVALID: + case shrexpb.Status_INVALID: log.Warn("client-nd: invalid request") fallthrough - case pb.StatusCode_INTERNAL: + case shrexpb.Status_INTERNAL: fallthrough default: return shrex.ErrInvalidResponse diff --git a/share/shwap/p2p/shrex/shrexnd/exchange_test.go b/share/shwap/p2p/shrex/shrexnd/exchange_test.go index 7de8f7c6ec..c49ebd867e 100644 --- a/share/shwap/p2p/shrex/shrexnd/exchange_test.go +++ b/share/shwap/p2p/shrex/shrexnd/exchange_test.go @@ -3,6 +3,7 @@ package shrexnd import ( "context" "sync" + "sync/atomic" "testing" "time" @@ -10,7 +11,6 @@ import ( "github.com/libp2p/go-libp2p/core/network" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" - "go.uber.org/atomic" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/edstest" @@ -25,7 +25,8 @@ func TestExchange_RequestND_NotFound(t *testing.T) { edsStore, client, server := makeExchange(t) require.NoError(t, server.Start(ctx)) - height := atomic.NewUint64(1) + height := atomic.Uint64{} + height.Add(1) t.Run("CAR_not_exist", func(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) diff --git a/share/shwap/p2p/shrex/shrexnd/params.go b/share/shwap/p2p/shrex/shrexnd/params.go index 2e1acd6010..32e2ba9c55 100644 --- a/share/shwap/p2p/shrex/shrexnd/params.go +++ b/share/shwap/p2p/shrex/shrexnd/params.go @@ -8,7 +8,7 @@ import ( "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" ) -const protocolString = "/shrex/nd/v0.0.4" +const protocolString = "/shrex/nd/v0.1.0" var log = logging.Logger("shrex/nd") diff --git a/share/shwap/p2p/shrex/shrexnd/pb/row_namespace_data.pb.go b/share/shwap/p2p/shrex/shrexnd/pb/row_namespace_data.pb.go deleted file mode 100644 index 071046a85a..0000000000 --- a/share/shwap/p2p/shrex/shrexnd/pb/row_namespace_data.pb.go +++ /dev/null @@ -1,576 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: share/shwap/p2p/shrex/shrexnd/pb/row_namespace_data.proto - -package share_p2p_shrex_nd - -import ( - fmt "fmt" - pb "github.com/celestiaorg/nmt/pb" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -type StatusCode int32 - -const ( - StatusCode_INVALID StatusCode = 0 - StatusCode_OK StatusCode = 1 - StatusCode_NOT_FOUND StatusCode = 2 - StatusCode_INTERNAL StatusCode = 3 -) - -var StatusCode_name = map[int32]string{ - 0: "INVALID", - 1: "OK", - 2: "NOT_FOUND", - 3: "INTERNAL", -} - -var StatusCode_value = map[string]int32{ - "INVALID": 0, - "OK": 1, - "NOT_FOUND": 2, - "INTERNAL": 3, -} - -func (x StatusCode) String() string { - return proto.EnumName(StatusCode_name, int32(x)) -} - -func (StatusCode) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_e8097b1aa3ae2e25, []int{0} -} - -type GetSharesByNamespaceStatusResponse struct { - Status StatusCode `protobuf:"varint,1,opt,name=status,proto3,enum=share.p2p.shrex.nd.StatusCode" json:"status,omitempty"` -} - -func (m *GetSharesByNamespaceStatusResponse) Reset() { *m = GetSharesByNamespaceStatusResponse{} } -func (m *GetSharesByNamespaceStatusResponse) String() string { return proto.CompactTextString(m) } -func (*GetSharesByNamespaceStatusResponse) ProtoMessage() {} -func (*GetSharesByNamespaceStatusResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e8097b1aa3ae2e25, []int{0} -} -func (m *GetSharesByNamespaceStatusResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *GetSharesByNamespaceStatusResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_GetSharesByNamespaceStatusResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *GetSharesByNamespaceStatusResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetSharesByNamespaceStatusResponse.Merge(m, src) -} -func (m *GetSharesByNamespaceStatusResponse) XXX_Size() int { - return m.Size() -} -func (m *GetSharesByNamespaceStatusResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetSharesByNamespaceStatusResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_GetSharesByNamespaceStatusResponse proto.InternalMessageInfo - -func (m *GetSharesByNamespaceStatusResponse) GetStatus() StatusCode { - if m != nil { - return m.Status - } - return StatusCode_INVALID -} - -type NamespaceRowResponse struct { - Shares [][]byte `protobuf:"bytes,1,rep,name=shares,proto3" json:"shares,omitempty"` - Proof *pb.Proof `protobuf:"bytes,2,opt,name=proof,proto3" json:"proof,omitempty"` -} - -func (m *NamespaceRowResponse) Reset() { *m = NamespaceRowResponse{} } -func (m *NamespaceRowResponse) String() string { return proto.CompactTextString(m) } -func (*NamespaceRowResponse) ProtoMessage() {} -func (*NamespaceRowResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e8097b1aa3ae2e25, []int{1} -} -func (m *NamespaceRowResponse) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *NamespaceRowResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_NamespaceRowResponse.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *NamespaceRowResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_NamespaceRowResponse.Merge(m, src) -} -func (m *NamespaceRowResponse) XXX_Size() int { - return m.Size() -} -func (m *NamespaceRowResponse) XXX_DiscardUnknown() { - xxx_messageInfo_NamespaceRowResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_NamespaceRowResponse proto.InternalMessageInfo - -func (m *NamespaceRowResponse) GetShares() [][]byte { - if m != nil { - return m.Shares - } - return nil -} - -func (m *NamespaceRowResponse) GetProof() *pb.Proof { - if m != nil { - return m.Proof - } - return nil -} - -func init() { - proto.RegisterEnum("share.p2p.shrex.nd.StatusCode", StatusCode_name, StatusCode_value) - proto.RegisterType((*GetSharesByNamespaceStatusResponse)(nil), "share.p2p.shrex.nd.GetSharesByNamespaceStatusResponse") - proto.RegisterType((*NamespaceRowResponse)(nil), "share.p2p.shrex.nd.NamespaceRowResponse") -} - -func init() { - proto.RegisterFile("share/shwap/p2p/shrex/shrexnd/pb/row_namespace_data.proto", fileDescriptor_e8097b1aa3ae2e25) -} - -var fileDescriptor_e8097b1aa3ae2e25 = []byte{ - // 301 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xb2, 0x2c, 0xce, 0x48, 0x2c, - 0x4a, 0xd5, 0x2f, 0xce, 0x28, 0x4f, 0x2c, 0xd0, 0x2f, 0x30, 0x2a, 0xd0, 0x2f, 0xce, 0x28, 0x4a, - 0xad, 0x80, 0x90, 0x79, 0x29, 0xfa, 0x05, 0x49, 0xfa, 0x45, 0xf9, 0xe5, 0xf1, 0x79, 0x89, 0xb9, - 0xa9, 0xc5, 0x05, 0x89, 0xc9, 0xa9, 0xf1, 0x29, 0x89, 0x25, 0x89, 0x7a, 0x05, 0x45, 0xf9, 0x25, - 0xf9, 0x42, 0x42, 0x60, 0xad, 0x7a, 0x05, 0x46, 0x05, 0x7a, 0x60, 0xe5, 0x7a, 0x79, 0x29, 0x52, - 0x7c, 0x05, 0x49, 0xfa, 0x05, 0x45, 0xf9, 0xf9, 0x69, 0x10, 0x35, 0x4a, 0x31, 0x5c, 0x4a, 0xee, - 0xa9, 0x25, 0xc1, 0x20, 0x85, 0xc5, 0x4e, 0x95, 0x7e, 0x30, 0x63, 0x82, 0x4b, 0x12, 0x4b, 0x4a, - 0x8b, 0x83, 0x52, 0x8b, 0x0b, 0xf2, 0xf3, 0x8a, 0x53, 0x85, 0xcc, 0xb8, 0xd8, 0x8a, 0xc1, 0x22, - 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x7c, 0x46, 0x72, 0x7a, 0x98, 0x46, 0xeb, 0x41, 0xf4, 0x38, 0xe7, - 0xa7, 0xa4, 0x06, 0x41, 0x55, 0x2b, 0x85, 0x72, 0x89, 0xc0, 0x8d, 0x0c, 0xca, 0x2f, 0x87, 0x9b, - 0x27, 0xc6, 0xc5, 0x06, 0x36, 0x00, 0x64, 0x1e, 0xb3, 0x06, 0x4f, 0x10, 0x94, 0x27, 0xa4, 0xca, - 0xc5, 0x0a, 0x76, 0x9c, 0x04, 0x93, 0x02, 0xa3, 0x06, 0xb7, 0x11, 0xbf, 0x1e, 0xd4, 0xa9, 0x49, - 0x7a, 0x01, 0x20, 0x46, 0x10, 0x44, 0x56, 0xcb, 0x8e, 0x8b, 0x0b, 0x61, 0x99, 0x10, 0x37, 0x17, - 0xbb, 0xa7, 0x5f, 0x98, 0xa3, 0x8f, 0xa7, 0x8b, 0x00, 0x83, 0x10, 0x1b, 0x17, 0x93, 0xbf, 0xb7, - 0x00, 0xa3, 0x10, 0x2f, 0x17, 0xa7, 0x9f, 0x7f, 0x48, 0xbc, 0x9b, 0x7f, 0xa8, 0x9f, 0x8b, 0x00, - 0x93, 0x10, 0x0f, 0x17, 0x87, 0xa7, 0x5f, 0x88, 0x6b, 0x90, 0x9f, 0xa3, 0x8f, 0x00, 0xb3, 0x93, - 0xc4, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, - 0x1c, 0xc3, 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, 0x31, 0x24, 0xb1, 0x81, 0x43, 0xc5, 0x18, - 0x10, 0x00, 0x00, 0xff, 0xff, 0x82, 0x3a, 0xfe, 0x72, 0x76, 0x01, 0x00, 0x00, -} - -func (m *GetSharesByNamespaceStatusResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *GetSharesByNamespaceStatusResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *GetSharesByNamespaceStatusResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Status != 0 { - i = encodeVarintRowNamespaceData(dAtA, i, uint64(m.Status)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func (m *NamespaceRowResponse) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *NamespaceRowResponse) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *NamespaceRowResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.Proof != nil { - { - size, err := m.Proof.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintRowNamespaceData(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - if len(m.Shares) > 0 { - for iNdEx := len(m.Shares) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Shares[iNdEx]) - copy(dAtA[i:], m.Shares[iNdEx]) - i = encodeVarintRowNamespaceData(dAtA, i, uint64(len(m.Shares[iNdEx]))) - i-- - dAtA[i] = 0xa - } - } - return len(dAtA) - i, nil -} - -func encodeVarintRowNamespaceData(dAtA []byte, offset int, v uint64) int { - offset -= sovRowNamespaceData(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *GetSharesByNamespaceStatusResponse) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Status != 0 { - n += 1 + sovRowNamespaceData(uint64(m.Status)) - } - return n -} - -func (m *NamespaceRowResponse) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if len(m.Shares) > 0 { - for _, b := range m.Shares { - l = len(b) - n += 1 + l + sovRowNamespaceData(uint64(l)) - } - } - if m.Proof != nil { - l = m.Proof.Size() - n += 1 + l + sovRowNamespaceData(uint64(l)) - } - return n -} - -func sovRowNamespaceData(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozRowNamespaceData(x uint64) (n int) { - return sovRowNamespaceData(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (m *GetSharesByNamespaceStatusResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRowNamespaceData - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: GetSharesByNamespaceStatusResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: GetSharesByNamespaceStatusResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) - } - m.Status = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRowNamespaceData - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Status |= StatusCode(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipRowNamespaceData(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthRowNamespaceData - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *NamespaceRowResponse) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRowNamespaceData - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: NamespaceRowResponse: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: NamespaceRowResponse: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Shares", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRowNamespaceData - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthRowNamespaceData - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthRowNamespaceData - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Shares = append(m.Shares, make([]byte, postIndex-iNdEx)) - copy(m.Shares[len(m.Shares)-1], dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Proof", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowRowNamespaceData - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthRowNamespaceData - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthRowNamespaceData - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Proof == nil { - m.Proof = &pb.Proof{} - } - if err := m.Proof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipRowNamespaceData(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthRowNamespaceData - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipRowNamespaceData(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - depth := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowRowNamespaceData - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowRowNamespaceData - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - case 1: - iNdEx += 8 - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowRowNamespaceData - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthRowNamespaceData - } - iNdEx += length - case 3: - depth++ - case 4: - if depth == 0 { - return 0, ErrUnexpectedEndOfGroupRowNamespaceData - } - depth-- - case 5: - iNdEx += 4 - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - if iNdEx < 0 { - return 0, ErrInvalidLengthRowNamespaceData - } - if depth == 0 { - return iNdEx, nil - } - } - return 0, io.ErrUnexpectedEOF -} - -var ( - ErrInvalidLengthRowNamespaceData = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowRowNamespaceData = fmt.Errorf("proto: integer overflow") - ErrUnexpectedEndOfGroupRowNamespaceData = fmt.Errorf("proto: unexpected end of group") -) diff --git a/share/shwap/p2p/shrex/shrexnd/pb/row_namespace_data.proto b/share/shwap/p2p/shrex/shrexnd/pb/row_namespace_data.proto deleted file mode 100644 index 4f484a8c2b..0000000000 --- a/share/shwap/p2p/shrex/shrexnd/pb/row_namespace_data.proto +++ /dev/null @@ -1,20 +0,0 @@ -syntax = "proto3"; - -package share.p2p.shrex.nd; -import "pb/proof.proto"; - -message GetSharesByNamespaceStatusResponse{ - StatusCode status = 1; -} - -enum StatusCode { - INVALID = 0; - OK = 1; - NOT_FOUND = 2; - INTERNAL = 3; -}; - -message NamespaceRowResponse { - repeated bytes shares = 1; - proof.pb.Proof proof = 2; -} diff --git a/share/shwap/p2p/shrex/shrexnd/server.go b/share/shwap/p2p/shrex/shrexnd/server.go index 784d459614..8aa76cf6a7 100644 --- a/share/shwap/p2p/shrex/shrexnd/server.go +++ b/share/shwap/p2p/shrex/shrexnd/server.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io" "time" "github.com/libp2p/go-libp2p/core/host" @@ -13,13 +12,12 @@ import ( "go.uber.org/zap" "github.com/celestiaorg/go-libp2p-messenger/serde" - nmt_pb "github.com/celestiaorg/nmt/pb" "github.com/celestiaorg/celestia-node/libs/utils" eds "github.com/celestiaorg/celestia-node/share/new_eds" "github.com/celestiaorg/celestia-node/share/shwap" "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex" - pb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/shrexnd/pb" + shrexpb "github.com/celestiaorg/celestia-node/share/shwap/p2p/shrex/pb" "github.com/celestiaorg/celestia-node/store" ) @@ -118,7 +116,7 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre ctx, cancel := context.WithTimeout(ctx, srv.params.HandleRequestTimeout) defer cancel() - shares, status, err := srv.getNamespaceData(ctx, ndid) + nd, status, err := srv.getNamespaceData(ctx, ndid) if err != nil { // server should respond with status regardless if there was an error getting data sendErr := srv.respondStatus(ctx, logger, stream, status) @@ -137,7 +135,7 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre return err } - err = srv.sendNamespacedShares(shares, stream) + _, err = nd.WriteTo(stream) if err != nil { logger.Errorw("send nd data", "err", err) srv.metrics.ObserveRequests(ctx, 1, shrex.StatusSendRespErr) @@ -155,15 +153,11 @@ func (srv *Server) readRequest( logger.Debugw("setting read deadline", "err", err) } - req := make([]byte, shwap.NamespaceDataIDSize) - _, err = io.ReadFull(stream, req) + ndid := shwap.NamespaceDataID{} + _, err = ndid.ReadFrom(stream) if err != nil { return shwap.NamespaceDataID{}, fmt.Errorf("reading request: %w", err) } - id, err := shwap.NamespaceDataIDFromBinary(req) - if err != nil { - return shwap.NamespaceDataID{}, fmt.Errorf("decoding request: %w", err) - } logger.Debugw("new request") err = stream.CloseRead() @@ -171,32 +165,32 @@ func (srv *Server) readRequest( logger.Debugw("closing read side of the stream", "err", err) } - return id, nil + return ndid, nil } -func (srv *Server) getNamespaceData(ctx context.Context, id shwap.NamespaceDataID) (shwap.NamespacedData, pb.StatusCode, error) { +func (srv *Server) getNamespaceData(ctx context.Context, id shwap.NamespaceDataID) (shwap.NamespacedData, shrexpb.Status, error) { file, err := srv.store.GetByHeight(ctx, id.Height) if errors.Is(err, store.ErrNotFound) { - return nil, pb.StatusCode_NOT_FOUND, nil + return nil, shrexpb.Status_NOT_FOUND, nil } if err != nil { - return nil, pb.StatusCode_INTERNAL, fmt.Errorf("retrieving DAH: %w", err) + return nil, shrexpb.Status_INTERNAL, fmt.Errorf("retrieving DAH: %w", err) } defer utils.CloseAndLog(log, "file", file) nd, err := eds.NamespacedData(ctx, file, id.DataNamespace) if err != nil { - return nil, pb.StatusCode_INTERNAL, fmt.Errorf("getting nd: %w", err) + return nil, shrexpb.Status_INVALID, fmt.Errorf("getting nd: %w", err) } - return nd, pb.StatusCode_OK, nil + return nd, shrexpb.Status_OK, nil } func (srv *Server) respondStatus( ctx context.Context, logger *zap.SugaredLogger, stream network.Stream, - status pb.StatusCode, + status shrexpb.Status, ) error { srv.observeStatus(ctx, status) @@ -205,7 +199,7 @@ func (srv *Server) respondStatus( logger.Debugw("setting write deadline", "err", err) } - _, err = serde.Write(stream, &pb.GetSharesByNamespaceStatusResponse{Status: status}) + _, err = serde.Write(stream, &shrexpb.Response{Status: status}) if err != nil { return fmt.Errorf("writing response: %w", err) } @@ -213,34 +207,13 @@ func (srv *Server) respondStatus( return nil } -// sendNamespacedShares encodes shares into proto messages and sends it to client -func (srv *Server) sendNamespacedShares(data shwap.NamespacedData, stream network.Stream) error { - for _, row := range data { - row := &pb.NamespaceRowResponse{ - Shares: row.Shares, - Proof: &nmt_pb.Proof{ - Start: int64(row.Proof.Start()), - End: int64(row.Proof.End()), - Nodes: row.Proof.Nodes(), - LeafHash: row.Proof.LeafHash(), - IsMaxNamespaceIgnored: row.Proof.IsMaxNamespaceIDIgnored(), - }, - } - _, err := serde.Write(stream, row) - if err != nil { - return fmt.Errorf("writing nd data to stream: %w", err) - } - } - return nil -} - -func (srv *Server) observeStatus(ctx context.Context, status pb.StatusCode) { +func (srv *Server) observeStatus(ctx context.Context, status shrexpb.Status) { switch { - case status == pb.StatusCode_OK: + case status == shrexpb.Status_OK: srv.metrics.ObserveRequests(ctx, 1, shrex.StatusSuccess) - case status == pb.StatusCode_NOT_FOUND: + case status != shrexpb.Status_NOT_FOUND: srv.metrics.ObserveRequests(ctx, 1, shrex.StatusNotFound) - case status == pb.StatusCode_INTERNAL: + case status == shrexpb.Status_INVALID: srv.metrics.ObserveRequests(ctx, 1, shrex.StatusInternalErr) } }