From ac2a626c951eff5927e5a33e32ca5e95a2cc5d69 Mon Sep 17 00:00:00 2001 From: Alexander Vykhodtsev Date: Fri, 2 Dec 2022 21:20:40 +0600 Subject: [PATCH 1/3] Update parquet lib to support running on Apple M1 --- build.gradle | 2 +- .../events/handling/BeamEventsWriterParquet.scala | 6 ++++-- src/main/scala/beam/router/r5/RouteDumper.scala | 8 ++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index a5d41ef0797..6c81c19cf78 100755 --- a/build.gradle +++ b/build.gradle @@ -311,7 +311,7 @@ dependencies { implementation group: 'com.zaxxer', name: 'nuprocess', version: '1.2.4' - def parquet = "1.10.0" + def parquet = "1.12.3" implementation group: 'org.apache.parquet', name: 'parquet-hadoop', version: parquet implementation group: 'org.apache.parquet', name: 'parquet-avro', version: parquet implementation(group: 'org.apache.hadoop', name: 'hadoop-client', version: '2.7.3') { diff --git a/src/main/scala/beam/agentsim/events/handling/BeamEventsWriterParquet.scala b/src/main/scala/beam/agentsim/events/handling/BeamEventsWriterParquet.scala index 655b8ad6499..c2d278017ae 100644 --- a/src/main/scala/beam/agentsim/events/handling/BeamEventsWriterParquet.scala +++ b/src/main/scala/beam/agentsim/events/handling/BeamEventsWriterParquet.scala @@ -10,6 +10,7 @@ import org.apache.hadoop.fs.Path import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.ParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName +import org.apache.parquet.hadoop.util.HadoopOutputFile import org.matsim.api.core.v01.events.Event import scala.collection.JavaConverters._ @@ -71,10 +72,11 @@ class BeamEventsWriterParquet( def getWriter(schema: Schema, filePath: String): ParquetWriter[GenericData.Record] = { val path = new Path(filePath) - val builder = AvroParquetWriter.builder[GenericData.Record](path) + val outputFile = HadoopOutputFile.fromPath(path, new Configuration()) + val builder = AvroParquetWriter.builder[GenericData.Record](outputFile) builder - .withRowGroupSize(ParquetWriter.DEFAULT_BLOCK_SIZE) + .withRowGroupSize(ParquetWriter.DEFAULT_BLOCK_SIZE.toLong) .withPageSize(ParquetWriter.DEFAULT_PAGE_SIZE) .withSchema(schema) .withConf(new Configuration()) diff --git a/src/main/scala/beam/router/r5/RouteDumper.scala b/src/main/scala/beam/router/r5/RouteDumper.scala index 6baca7b369e..295b7a84e32 100644 --- a/src/main/scala/beam/router/r5/RouteDumper.scala +++ b/src/main/scala/beam/router/r5/RouteDumper.scala @@ -1,7 +1,6 @@ package beam.router.r5 import java.util - import beam.agentsim.agents.vehicles.VehicleProtocol.StreetVehicle import beam.agentsim.events.SpaceTime import beam.router.BeamRouter.{EmbodyWithCurrentTravelTime, RoutingRequest, RoutingResponse} @@ -11,10 +10,12 @@ import beam.sim.population.{AttributesOfIndividual, HouseholdAttributes} import org.apache.avro.Schema.Type import org.apache.avro.generic.GenericData import org.apache.avro.{Schema, SchemaBuilder} +import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.ParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName +import org.apache.parquet.hadoop.util.HadoopOutputFile import org.matsim.api.core.v01.events.Event import org.matsim.core.controler.OutputDirectoryHierarchy import org.matsim.core.controler.events.{IterationEndsEvent, IterationStartsEvent} @@ -555,10 +556,9 @@ object RouteDumper { } def createWriter(path: String, schema: Schema): ParquetWriter[GenericData.Record] = { + val outputFile = HadoopOutputFile.fromPath(new Path(path), new Configuration()) AvroParquetWriter - .builder[GenericData.Record]( - new Path(path) - ) + .builder[GenericData.Record](outputFile) .withSchema(schema) .withCompressionCodec(CompressionCodecName.SNAPPY) .build() From 49f5190c64fb32521cc6b5d58cb4750ecfd0451f Mon Sep 17 00:00:00 2001 From: Alexander Vykhodtsev Date: Sat, 3 Dec 2022 14:33:09 +0600 Subject: [PATCH 2/3] Use native Google ORTools library loader since it is now available in maven --- build.gradle | 4 +--- .../agents/ridehail/AlonsoMoraMatchingWithMIPAssignment.scala | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 6c81c19cf78..550c0775f5a 100755 --- a/build.gradle +++ b/build.gradle @@ -179,9 +179,7 @@ dependencies { implementation "org.jgrapht:jgrapht-core:1.3.0" - implementation('com.github.LBNL-UCB-STI:or-tools-wrapper:7.5-0') { - exclude group: 'com.google.protobuf', module: 'protobuf-java' - } + implementation group: 'com.google.ortools', name: 'ortools-java', version: '9.5.2237' implementation 'com.github.LBNL-UCB-STI:helics-wrapper:v3.3.0' diff --git a/src/main/scala/beam/agentsim/agents/ridehail/AlonsoMoraMatchingWithMIPAssignment.scala b/src/main/scala/beam/agentsim/agents/ridehail/AlonsoMoraMatchingWithMIPAssignment.scala index 12ef3bac4d9..c0a1c2489e0 100644 --- a/src/main/scala/beam/agentsim/agents/ridehail/AlonsoMoraMatchingWithMIPAssignment.scala +++ b/src/main/scala/beam/agentsim/agents/ridehail/AlonsoMoraMatchingWithMIPAssignment.scala @@ -4,7 +4,7 @@ import beam.agentsim.agents.MobilityRequest import beam.agentsim.agents.ridehail.RideHailMatching._ import beam.sim.BeamServices import beam.sim.config.BeamConfig.Beam.Agentsim.Agents.RideHail.Managers$Elm -import com.github.beam.OrToolsLoader +import com.google.ortools.Loader import com.google.ortools.linearsolver.{MPSolver, MPVariable} import org.jgrapht.graph.DefaultEdge import org.matsim.core.utils.collections.QuadTree @@ -17,7 +17,7 @@ import scala.concurrent.Future object AlonsoMoraMatchingWithMIPAssignment { private lazy val initialize: Unit = { - OrToolsLoader.load() + Loader.loadNativeLibraries() } } From 043c9f3d1ee9981e27dfc5a79a153c18a3e468c7 Mon Sep 17 00:00:00 2001 From: Alexander Vykhodtsev Date: Wed, 7 Dec 2022 23:30:31 +0600 Subject: [PATCH 3/3] Try ortools 9.4 since 9.5 requires GLIBC 2.32 which is not available on CI --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 550c0775f5a..6f4172615dc 100755 --- a/build.gradle +++ b/build.gradle @@ -179,7 +179,7 @@ dependencies { implementation "org.jgrapht:jgrapht-core:1.3.0" - implementation group: 'com.google.ortools', name: 'ortools-java', version: '9.5.2237' + implementation group: 'com.google.ortools', name: 'ortools-java', version: '9.4.1874' implementation 'com.github.LBNL-UCB-STI:helics-wrapper:v3.3.0'