Synthetic PoRep (#400) (#418)

* feat: add various new APIs for Lotus Those new APIs are about: - decoding a range of an updated empty sector - generating TreeRLast - generating TreeC - running SDR * docs: document the new APIs Add API documentation to the new APIs. Also change one parameter name for more consistency. * wip Go bindings for new proving APIs * Add nicer wrapper for GenerateSDR * feat: update proofs to the latest release * cleanup comments in proofs.go * feat: update dependencies to WIP synthetic-porep code * feat: add example of using SynthPoRep in existing test * feat: wire in new APIs * feat: add clear_layer_data API and more robustly use synth-porep in test * feat: go-wrapper for ClearSyntheticProofs * merged synth work * Update deps after (messy) rebase * missing v21 engine mapping * newer cargo lock * feat: point to fvm master after synthporep changes landed * update the fvm & bls-signatures (to match the fvm) --------- Co-authored-by: Volker Mische <[email protected]> Co-authored-by: Łukasz Magiera <[email protected]> Co-authored-by: Aayush <[email protected]> Co-authored-by: Andrew Jackson (Ajax) <[email protected]> Co-authored-by: Steven Allen <[email protected]>
filecoin-project · Sep 7, 2023 · 310f0cc · 310f0cc
1 parent bf5edd5
commit 310f0cc
Show file tree

Hide file tree

Showing 9 changed files with 605 additions and 789 deletions.
diff --git a/cgo/const.go b/cgo/const.go
@@ -16,16 +16,21 @@ const (
 )
 
 const (
-	RegisteredSealProofStackedDrg2KiBV1    = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1
-	RegisteredSealProofStackedDrg8MiBV1    = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1
-	RegisteredSealProofStackedDrg512MiBV1  = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1
-	RegisteredSealProofStackedDrg32GiBV1   = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1
-	RegisteredSealProofStackedDrg64GiBV1   = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1
-	RegisteredSealProofStackedDrg2KiBV11   = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1_1
-	RegisteredSealProofStackedDrg8MiBV11   = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1_1
-	RegisteredSealProofStackedDrg512MiBV11 = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1_1
-	RegisteredSealProofStackedDrg32GiBV11  = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1_1
-	RegisteredSealProofStackedDrg64GiBV11  = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1_1
+	RegisteredSealProofStackedDrg2KiBV1                        = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1
+	RegisteredSealProofStackedDrg8MiBV1                        = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1
+	RegisteredSealProofStackedDrg512MiBV1                      = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1
+	RegisteredSealProofStackedDrg32GiBV1                       = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1
+	RegisteredSealProofStackedDrg64GiBV1                       = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1
+	RegisteredSealProofStackedDrg2KiBV11                       = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1_1
+	RegisteredSealProofStackedDrg8MiBV11                       = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1_1
+	RegisteredSealProofStackedDrg512MiBV11                     = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1_1
+	RegisteredSealProofStackedDrg32GiBV11                      = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1_1
+	RegisteredSealProofStackedDrg64GiBV11                      = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1_1
+	RegisteredSealProofStackedDrg2KiBV11_Feat_SyntheticPoRep   = C.REGISTERED_SEAL_PROOF_STACKED_DRG2_KI_B_V1_1__FEAT__SYNTHETIC_PO_REP
+	RegisteredSealProofStackedDrg8MiBV11_Feat_SyntheticPoRep   = C.REGISTERED_SEAL_PROOF_STACKED_DRG8_MI_B_V1_1__FEAT__SYNTHETIC_PO_REP
+	RegisteredSealProofStackedDrg512MiBV11_Feat_SyntheticPoRep = C.REGISTERED_SEAL_PROOF_STACKED_DRG512_MI_B_V1_1__FEAT__SYNTHETIC_PO_REP
+	RegisteredSealProofStackedDrg32GiBV11_Feat_SyntheticPoRep  = C.REGISTERED_SEAL_PROOF_STACKED_DRG32_GI_B_V1_1__FEAT__SYNTHETIC_PO_REP
+	RegisteredSealProofStackedDrg64GiBV11_Feat_SyntheticPoRep  = C.REGISTERED_SEAL_PROOF_STACKED_DRG64_GI_B_V1_1__FEAT__SYNTHETIC_PO_REP
 )
 
 const (

diff --git a/cgo/proofs.go b/cgo/proofs.go
@@ -224,6 +224,39 @@ func ClearCache(sectorSize uint64, cacheDirPath SliceRefUint8) error {
 	return CheckErr(resp)
 }
 
+func ClearSyntheticProofs(sectorSize uint64, cacheDirPath SliceRefUint8) error {
+	resp := C.clear_synthetic_proofs(C.uint64_t(sectorSize), cacheDirPath)
+	defer resp.destroy()
+	return CheckErr(resp)
+}
+func ClearLayerData(sectorSize uint64, cacheDirPath SliceRefUint8) error {
+	resp := C.clear_layer_data(C.uint64_t(sectorSize), cacheDirPath)
+	defer resp.destroy()
+	return CheckErr(resp)
+}
+
+func GenerateSynthProofs(
+	registered_proof RegisteredSealProof,
+	comm_r, comm_d ByteArray32,
+	cache_dir_path, replica_path SliceRefUint8,
+	sector_id uint64,
+	prover_id, ticket ByteArray32,
+	pieces SliceRefPublicPieceInfo,
+) error {
+	resp := C.generate_synth_proofs(registered_proof,
+		&comm_r,
+		&comm_d,
+		cache_dir_path,
+		replica_path,
+		C.uint64_t(sector_id),
+		&prover_id,
+		&ticket,
+		pieces,
+	)
+	defer resp.destroy()
+	return CheckErr(resp)
+}
+
 func Fauxrep(registeredProf RegisteredSealProof, cacheDirPath SliceRefUint8, sealedSectorPath SliceRefUint8) ([]byte, error) {
 	resp := C.fauxrep(registeredProf, cacheDirPath, sealedSectorPath)
 	defer resp.destroy()

diff --git a/go.mod b/go.mod
@@ -5,7 +5,7 @@ go 1.18
 require (
 	github.com/filecoin-project/go-address v1.1.0
 	github.com/filecoin-project/go-fil-commcid v0.1.0
-	github.com/filecoin-project/go-state-types v0.11.1
+	github.com/filecoin-project/go-state-types v0.11.2-0.20230712101859-8f37624fa540
 	github.com/ipfs/go-block-format v0.0.3
 	github.com/ipfs/go-cid v0.3.2
 	github.com/ipfs/go-ipfs-blockstore v1.2.0

diff --git a/go.sum b/go.sum
@@ -28,8 +28,8 @@ github.com/filecoin-project/go-state-types v0.0.0-20200903145444-247639ffa6ad/go
 github.com/filecoin-project/go-state-types v0.0.0-20200904021452-1883f36ca2f4/go.mod h1:IQ0MBPnonv35CJHtWSN3YY1Hz2gkPru1Q9qoaYLxx9I=
 github.com/filecoin-project/go-state-types v0.0.0-20201102161440-c8033295a1fc/go.mod h1:ezYnPf0bNkTsDibL/psSz5dy4B5awOJ/E7P2Saeep8g=
 github.com/filecoin-project/go-state-types v0.1.10/go.mod h1:UwGVoMsULoCK+bWjEdd/xLCvLAQFBC7EDT477SKml+Q=
-github.com/filecoin-project/go-state-types v0.11.1 h1:GDtCN9V18bYVwXDZe+vJXc6Ck+qY9OUaQqpoVlp1FAk=
-github.com/filecoin-project/go-state-types v0.11.1/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8=
+github.com/filecoin-project/go-state-types v0.11.2-0.20230712101859-8f37624fa540 h1:v0fbEnBrMIjlxLve1sJTJE0YNGg58SNiP5sxQtr3trc=
+github.com/filecoin-project/go-state-types v0.11.2-0.20230712101859-8f37624fa540/go.mod h1:SyNPwTsU7I22gL2r0OAPcImvLoTVfgRwdK/Y5rR1zz8=
 github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4=
 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=

diff --git a/proofs.go b/proofs.go
@@ -661,6 +661,53 @@ func ClearCache(sectorSize uint64, cacheDirPath string) error {
 	return cgo.ClearCache(sectorSize, cgo.AsSliceRefUint8([]byte(cacheDirPath)))
 }
 
+// ClearSyntheticProofs
+func ClearSyntheticProofs(sectorSize uint64, cacheDirPath string) error {
+	return cgo.ClearSyntheticProofs(sectorSize, cgo.AsSliceRefUint8([]byte(cacheDirPath)))
+}
+
+func ClearLayerData(sectorSize abi.SectorSize, cacheDirPath string) error {
+	return cgo.ClearLayerData(uint64(sectorSize), cgo.AsSliceRefUint8([]byte(cacheDirPath)))
+}
+
+func GenerateSynthProofs(
+	proofType abi.RegisteredSealProof,
+	sealedCID, unsealedCID cid.Cid,
+	cacheDirPath, replicaPath string,
+	sector_id abi.SectorNumber,
+	minerID abi.ActorID,
+	ticket []byte,
+	pieces []abi.PieceInfo,
+) error {
+	sp, err := toFilRegisteredSealProof(proofType)
+	if err != nil {
+		return err
+	}
+	filPublicPieceInfos, err := toFilPublicPieceInfos(pieces)
+	if err != nil {
+		return err
+	}
+	commR, err := to32ByteCommR(sealedCID)
+	if err != nil {
+		return err
+	}
+
+	commD, err := to32ByteCommD(unsealedCID)
+	if err != nil {
+		return err
+	}
+	proverID, err := toProverID(minerID)
+	if err != nil {
+		return err
+	}
+	return cgo.GenerateSynthProofs(sp,
+		commR, commD,
+		cgo.AsSliceRefUint8([]byte(cacheDirPath)), cgo.AsSliceRefUint8([]byte(replicaPath)),
+		uint64(sector_id),
+		proverID, cgo.AsByteArray32(ticket),
+		cgo.AsSliceRefPublicPieceInfo(filPublicPieceInfos))
+}
+
 func FauxRep(proofType abi.RegisteredSealProof, cacheDirPath string, sealedSectorPath string) (cid.Cid, error) {
 	sp, err := toFilRegisteredSealProof(proofType)
 	if err != nil {
@@ -975,6 +1022,18 @@ func toFilRegisteredSealProof(p abi.RegisteredSealProof) (cgo.RegisteredSealProo
 		return cgo.RegisteredSealProofStackedDrg32GiBV11, nil
 	case abi.RegisteredSealProof_StackedDrg64GiBV1_1:
 		return cgo.RegisteredSealProofStackedDrg64GiBV11, nil
+
+	case abi.RegisteredSealProof_StackedDrg2KiBV1_1_Feat_SyntheticPoRep:
+		return cgo.RegisteredSealProofStackedDrg2KiBV11_Feat_SyntheticPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg8MiBV1_1_Feat_SyntheticPoRep:
+		return cgo.RegisteredSealProofStackedDrg8MiBV11_Feat_SyntheticPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg512MiBV1_1_Feat_SyntheticPoRep:
+		return cgo.RegisteredSealProofStackedDrg512MiBV11_Feat_SyntheticPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg32GiBV1_1_Feat_SyntheticPoRep:
+		return cgo.RegisteredSealProofStackedDrg32GiBV11_Feat_SyntheticPoRep, nil
+	case abi.RegisteredSealProof_StackedDrg64GiBV1_1_Feat_SyntheticPoRep:
+		return cgo.RegisteredSealProofStackedDrg64GiBV11_Feat_SyntheticPoRep, nil
+
 	default:
 		return 0, errors.Errorf("no mapping to C.FFIRegisteredSealProof value available for: %v", p)
 	}