From d938b04f3bb36d581545eef08343a42806db6e5a Mon Sep 17 00:00:00 2001 From: frievoe97 Date: Wed, 6 Nov 2024 10:58:54 +0100 Subject: [PATCH] added activity dashboard and activity analysis --- .../activity/ActivityCountAnalysis.java | 163 ++++++++++++++++++ .../application/options/ShpOptions.java | 1 + .../main/java/org/matsim/simwrapper/Data.java | 12 ++ .../dashboard/ActivityDashboard.java | 36 ++++ .../simwrapper/dashboard/DashboardTests.java | 7 + .../src/test/resources/kehlheim_test.cpg | 1 + .../src/test/resources/kehlheim_test.dbf | Bin 0 -> 154 bytes .../src/test/resources/kehlheim_test.prj | 1 + .../src/test/resources/kehlheim_test.qmd | 27 +++ .../src/test/resources/kehlheim_test.shp | Bin 0 -> 2788 bytes .../src/test/resources/kehlheim_test.shx | Bin 0 -> 164 bytes 11 files changed, 248 insertions(+) create mode 100644 contribs/application/src/main/java/org/matsim/application/analysis/activity/ActivityCountAnalysis.java create mode 100644 contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/ActivityDashboard.java create mode 100644 contribs/simwrapper/src/test/resources/kehlheim_test.cpg create mode 100644 contribs/simwrapper/src/test/resources/kehlheim_test.dbf create mode 100644 contribs/simwrapper/src/test/resources/kehlheim_test.prj create mode 100644 contribs/simwrapper/src/test/resources/kehlheim_test.qmd create mode 100644 contribs/simwrapper/src/test/resources/kehlheim_test.shp create mode 100644 contribs/simwrapper/src/test/resources/kehlheim_test.shx diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/activity/ActivityCountAnalysis.java b/contribs/application/src/main/java/org/matsim/application/analysis/activity/ActivityCountAnalysis.java new file mode 100644 index 00000000000..3dfadf1a325 --- /dev/null +++ b/contribs/application/src/main/java/org/matsim/application/analysis/activity/ActivityCountAnalysis.java @@ -0,0 +1,163 @@ +package org.matsim.application.analysis.activity; + +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.Object2IntMap; +import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.geotools.api.feature.Feature; +import org.geotools.api.feature.Property; +import org.matsim.api.core.v01.Coord; +import org.matsim.application.CommandSpec; +import org.matsim.application.MATSimAppCommand; +import org.matsim.application.options.*; +import org.matsim.core.utils.io.IOUtils; +import picocli.CommandLine; +import tech.tablesaw.api.*; +import tech.tablesaw.io.csv.CsvReadOptions; +import tech.tablesaw.selection.Selection; + +import java.util.*; +import java.util.regex.Pattern; + +@CommandSpec( + requires = {"activities.csv"}, + produces = {"activities_%s_per_region.csv"} +) +public class ActivityCountAnalysis implements MATSimAppCommand { + + private static final Logger log = LogManager.getLogger(ActivityCountAnalysis.class); + + @CommandLine.Mixin + private final InputOptions input = InputOptions.ofCommand(ActivityCountAnalysis.class); + @CommandLine.Mixin + private final OutputOptions output = OutputOptions.ofCommand(ActivityCountAnalysis.class); + @CommandLine.Mixin + private ShpOptions shp; + @CommandLine.Mixin + private SampleOptions sample; + @CommandLine.Option(names = "--id-column", description = "Column to use as ID for the shapefile", required = true) + private String idColumn; + + @CommandLine.Option(names = "--activity-mapping", description = "Map of patterns to merge activity types", split = ";") + private Map activityMapping; + + public static void main(String[] args) { + new ActivityCountAnalysis().execute(args); + } + + @Override + public Integer call() throws Exception { + + HashMap> formattedActivityMapping = new HashMap<>(); + + if (this.activityMapping == null) this.activityMapping = new HashMap<>(); + + for (Map.Entry entry : this.activityMapping.entrySet()) { + String pattern = entry.getKey(); + String activity = entry.getValue(); + Set activities = new HashSet<>(Arrays.asList(activity.split(","))); + formattedActivityMapping.put(pattern, activities); + } + + // Reading the input csv + Table activities = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(input.getPath("activities.csv"))) + .columnTypesPartial(Map.of("person", ColumnType.TEXT, "activity_type", ColumnType.TEXT)) + .sample(false) + .separator(CsvOptions.detectDelimiter(input.getPath("activities.csv"))).build()); + + // remove the underscore and the number from the activity_type column + TextColumn activityType = activities.textColumn("activity_type"); + activityType.set(Selection.withRange(0, activityType.size()), activityType.replaceAll("_[0-9]{2,}$", "")); + + ShpOptions.Index index = shp.createIndex(idColumn); + + + // stores the counts of activities per region + Object2ObjectOpenHashMap> regionActivityCounts = new Object2ObjectOpenHashMap<>(); + // stores the activities that have been counted for each person in each region + Object2ObjectOpenHashMap> personActivityTracker = new Object2ObjectOpenHashMap<>(); + + // iterate over the csv rows + for (Row row : activities) { + String person = row.getString("person"); + String activity = row.getText("activity_type"); + + for (Map.Entry> entry : formattedActivityMapping.entrySet()) { + String pattern = entry.getKey(); + Set activities2 = entry.getValue(); + for (String act : activities2) { + if (Pattern.matches(act, activity)) { + activity = pattern; + break; + } + } + } + + Coord coord = new Coord(row.getDouble("coord_x"), row.getDouble("coord_y")); + + // get the region for the current coordinate + Feature feature = index.queryFeature(coord); + + if (feature == null) { + continue; + } + + Property prop = feature.getProperty(idColumn); + if (prop == null) + throw new IllegalArgumentException("No property found for column %s".formatted(idColumn)); + + Object region = prop.getValue(); + if (region != null && region.toString().length() > 0) { + + // Add region to the activity counts and person activity tracker if not already present + regionActivityCounts.computeIfAbsent(region, k -> new Object2IntOpenHashMap<>()); + personActivityTracker.computeIfAbsent(region, k -> new HashSet<>()); + + Set trackedActivities = personActivityTracker.get(region); + String personActivityKey = person + "_" + activity; + + // adding activity only if it has not been counted for the person in the region + if (!trackedActivities.contains(personActivityKey)) { + Object2IntMap activityCounts = regionActivityCounts.get(region); + activityCounts.mergeInt(activity, 1, Integer::sum); + + // mark the activity as counted for the person in the region + trackedActivities.add(personActivityKey); + } + } + } + + Set uniqueActivities = new HashSet<>(); + + for (Object2IntMap map : regionActivityCounts.values()) { + uniqueActivities.addAll(map.keySet()); + } + + for (String activity : uniqueActivities) { + Table resultTable = Table.create(); + TextColumn regionColumn = TextColumn.create("region"); + DoubleColumn activityColumn = DoubleColumn.create("count"); + resultTable.addColumns(regionColumn, activityColumn); + for (Map.Entry> entry : regionActivityCounts.entrySet()) { + Object region = entry.getKey(); + double value = 0; + for (Map.Entry entry2 : entry.getValue().object2IntEntrySet()) { + String ect = entry2.getKey(); + if (Pattern.matches(ect, activity)) { + value = entry2.getValue() * sample.getUpscaleFactor(); + break; + } + } + Row row = resultTable.appendRow(); + row.setString("region", region.toString()); + row.setDouble("count", value); + } + resultTable.addColumns(activityColumn.divide(activityColumn.sum()).setName("count_normalized")); + resultTable.write().csv(output.getPath("activities_%s_per_region.csv", activity).toFile()); + log.info("Wrote activity counts for {} to {}", activity, output.getPath("activities_%s_per_region.csv", activity)); + } + + return 0; + } +} diff --git a/contribs/application/src/main/java/org/matsim/application/options/ShpOptions.java b/contribs/application/src/main/java/org/matsim/application/options/ShpOptions.java index 5d9f8ccec99..55b7e01846e 100644 --- a/contribs/application/src/main/java/org/matsim/application/options/ShpOptions.java +++ b/contribs/application/src/main/java/org/matsim/application/options/ShpOptions.java @@ -230,6 +230,7 @@ public Geometry getGeometry() { /** * Return the union of all geometries in the shape file and project it to the target crs. + * * @param toCRS target coordinate system */ public Geometry getGeometry(String toCRS) { diff --git a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/Data.java b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/Data.java index d13bbc48ae4..07cdbdc43f3 100644 --- a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/Data.java +++ b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/Data.java @@ -1,5 +1,6 @@ package org.matsim.simwrapper; +import com.google.common.io.Resources; import org.apache.commons.io.FilenameUtils; import org.matsim.application.CommandRunner; import org.matsim.application.MATSimAppCommand; @@ -186,6 +187,17 @@ public String resource(String name) { return this.getUnixPath(this.path.getParent().relativize(resolved)); } + public String resources(String... names) { + String first = null; + for (String name : names) { + String resource = resource(name); + if (first == null) + first = resource; + } + + return first; + } + /** * Copies an input file (which can be local or an url) to the output directory. This method is intended to copy input for analysis classes. */ diff --git a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/ActivityDashboard.java b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/ActivityDashboard.java new file mode 100644 index 00000000000..b51a24347f5 --- /dev/null +++ b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/ActivityDashboard.java @@ -0,0 +1,36 @@ +package org.matsim.simwrapper.dashboard; + +import org.matsim.application.analysis.activity.ActivityCountAnalysis; +import org.matsim.simwrapper.Dashboard; +import org.matsim.simwrapper.Header; +import org.matsim.simwrapper.Layout; +import org.matsim.simwrapper.viz.MapPlot; + +public class ActivityDashboard implements Dashboard { + @Override + public void configure(Header header, Layout layout) { + + header.title = "Activity Analysis"; + header.description = "Displays the activities by type and location."; + + layout.row("activites") + .el(MapPlot.class, (viz, data) -> { + viz.title = "Activity Map"; + viz.description = "Activities per region"; + viz.height = 12.0; + viz.center = data.context().getCenter(); + viz.zoom = data.context().mapZoomLevel; + viz.display.fill.dataset = "activities_home"; + + // --shp= + data.resource(fff.shp) + + viz.display.fill.columnName = "count"; + String shp = data.resources("kehlheim_test.shp", "kehlheim_test.shx", "kehlheim_test.dbf", "kehlheim_test.prj"); + viz.setShape(shp); + + + viz.addDataset("activities_home", data.computeWithPlaceholder(ActivityCountAnalysis.class, "activities_%s_per_region.csv", "home", "--id-column=id", "--shp=" + shp)); + viz.addDataset("activities_other", data.computeWithPlaceholder(ActivityCountAnalysis.class, "activities_%s_per_region.csv", "other", "--id-column=id", "--shp=" + shp)); + }); + } +} diff --git a/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/DashboardTests.java b/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/DashboardTests.java index dca4a209df1..e28fd2d459b 100644 --- a/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/DashboardTests.java +++ b/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/DashboardTests.java @@ -178,4 +178,11 @@ void ptCustom() { run(pt); } + + @Test + void activity() { + ActivityDashboard ad = new ActivityDashboard(); + + run(ad); + } } diff --git a/contribs/simwrapper/src/test/resources/kehlheim_test.cpg b/contribs/simwrapper/src/test/resources/kehlheim_test.cpg new file mode 100644 index 00000000000..3ad133c048f --- /dev/null +++ b/contribs/simwrapper/src/test/resources/kehlheim_test.cpg @@ -0,0 +1 @@ +UTF-8 \ No newline at end of file diff --git a/contribs/simwrapper/src/test/resources/kehlheim_test.dbf b/contribs/simwrapper/src/test/resources/kehlheim_test.dbf new file mode 100644 index 0000000000000000000000000000000000000000..c13bf61d766d86188edadb889d7efbe15c57fa3a GIT binary patch literal 154 zcmZRs;gaKEU|?`$;0BVIATtFn<_BVN!MP9yuL2wxz*!202&NH&X^dc+Aeg2IrWt~1 GE(HK9l?~(o literal 0 HcmV?d00001 diff --git a/contribs/simwrapper/src/test/resources/kehlheim_test.prj b/contribs/simwrapper/src/test/resources/kehlheim_test.prj new file mode 100644 index 00000000000..bd846aeb220 --- /dev/null +++ b/contribs/simwrapper/src/test/resources/kehlheim_test.prj @@ -0,0 +1 @@ +PROJCS["ETRS_1989_UTM_Zone_32N",GEOGCS["GCS_ETRS_1989",DATUM["D_ETRS_1989",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",9.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]] \ No newline at end of file diff --git a/contribs/simwrapper/src/test/resources/kehlheim_test.qmd b/contribs/simwrapper/src/test/resources/kehlheim_test.qmd new file mode 100644 index 00000000000..087bef5cdd2 --- /dev/null +++ b/contribs/simwrapper/src/test/resources/kehlheim_test.qmd @@ -0,0 +1,27 @@ + + + + + + dataset + + + + + + + + + PROJCRS["ETRS89 / UTM zone 32N",BASEGEOGCRS["ETRS89",ENSEMBLE["European Terrestrial Reference System 1989 ensemble",MEMBER["European Terrestrial Reference Frame 1989"],MEMBER["European Terrestrial Reference Frame 1990"],MEMBER["European Terrestrial Reference Frame 1991"],MEMBER["European Terrestrial Reference Frame 1992"],MEMBER["European Terrestrial Reference Frame 1993"],MEMBER["European Terrestrial Reference Frame 1994"],MEMBER["European Terrestrial Reference Frame 1996"],MEMBER["European Terrestrial Reference Frame 1997"],MEMBER["European Terrestrial Reference Frame 2000"],MEMBER["European Terrestrial Reference Frame 2005"],MEMBER["European Terrestrial Reference Frame 2014"],ELLIPSOID["GRS 1980",6378137,298.257222101,LENGTHUNIT["metre",1]],ENSEMBLEACCURACY[0.1]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],ID["EPSG",4258]],CONVERSION["UTM zone 32N",METHOD["Transverse Mercator",ID["EPSG",9807]],PARAMETER["Latitude of natural origin",0,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8801]],PARAMETER["Longitude of natural origin",9,ANGLEUNIT["degree",0.0174532925199433],ID["EPSG",8802]],PARAMETER["Scale factor at natural origin",0.9996,SCALEUNIT["unity",1],ID["EPSG",8805]],PARAMETER["False easting",500000,LENGTHUNIT["metre",1],ID["EPSG",8806]],PARAMETER["False northing",0,LENGTHUNIT["metre",1],ID["EPSG",8807]]],CS[Cartesian,2],AXIS["(E)",east,ORDER[1],LENGTHUNIT["metre",1]],AXIS["(N)",north,ORDER[2],LENGTHUNIT["metre",1]],USAGE[SCOPE["Engineering survey, topographic mapping."],AREA["Europe between 6°E and 12°E: Austria; Belgium; Denmark - onshore and offshore; Germany - onshore and offshore; Norway including - onshore and offshore; Spain - offshore."],BBOX[38.76,6,84.33,12]],ID["EPSG",25832]] + +proj=utm +zone=32 +ellps=GRS80 +towgs84=0,0,0,0,0,0,0 +units=m +no_defs + 2105 + 25832 + EPSG:25832 + ETRS89 / UTM zone 32N + utm + EPSG:7019 + false + + + + diff --git a/contribs/simwrapper/src/test/resources/kehlheim_test.shp b/contribs/simwrapper/src/test/resources/kehlheim_test.shp new file mode 100644 index 0000000000000000000000000000000000000000..e9fb03fa9b2f604e47d38654bd0b67ff1e223795 GIT binary patch literal 2788 zcmZvedpwot7RNEPcWz0@5#5|8RETCuD4xoM(rsKSN@i=Ka%*Im6LUs}lH6wOqm&`! zx?R&YZI{SB3dxj4IJtGzOk|ME@11krk8^zH{d}H3zI(0pK5PA+^{)MhiD^lT{pBNd za_A#5F)7&oZvLC`k3SOnLxe!iCdZa`Cm9+|KO(g5;6fRWu(7eD*BmuP zV13!U!)fU=u@)DiWhOER&F|H$DuunrvGBs>Y(jmX2c-zWL5WgV+;a*2`ncD&^I%Pr zir(~mLbG^FvK+x|E_tz9Q3l`!T^G-4*b9_6WsAUHIk;!a=FQ~3*F2POi1%$%*uLxm zYPmXGGcc7ysQk6CEkj@>wY@TJ*#C5LK-_z9%Y^*0C0T?9?+o6|#rrNl**xKid_NR~ zHc5k*?tUk50bf+B@KXgxOWgI9fIt60sLx99p~#C9jhTf0a?Wy{HF&l@kWB)sgwgq5 zb(|4-qy;~lC31PHO6C-sg2QD35$7H-==w4ggKM-=CZ^h-etTfx64wkfkk}fPJRK4iqf5(9Tc|Wt?G!JmJNvKOx zIia2dsn$Ee(o-&@R;BQlU!1xL{Ot?t2Dh6tTFL)5spp4$j%wY#Y6bi4@Sek5)bmp^ zPsHRmTQ5k6FGVb#(LE!w^nwG*#{(p2=fFJX)zIyKm z^ef@pf9_SnAY$c>sw#)T`p-B666i0_9d^CL;DE?`Ew^x{6JKUnz6Xm*aZi+>KXuR4 z#6AJ9o+B`s0>@oyseT3KHdS*ZkhghLcZdY`ZrD6&)()I#0Du%+Tk6`}si)7lP$&j?aJ*whkQe0`Bn8~l&(M6;R(LeD-t_*Mzb=GOg!XHkb- z2e()rQK%zyDxKey4SuGPv_2jE_`Tr@`!MhqdY{bMfPQ0Z;*_8;ddU?0O*u(`tX}w$!`vogfvpJ#k)KFTk1bJM?T& zd!_<6Z3MCZt6SIN!DoqSgnpK3{p=QOu6=Sn0?ZrM*WGW^4QtG=V3X$&U9cC6WSkb_ z{Pg*pxJ0m;a%?Q~-FJH46+aAS_d0W{c6uXNl$PlogF3j5pBA zN3LHE>Xerg~*(+0Po1liWOGR?8WY_(;A&%j9gB8_*iX# zoz3l7&*bh0=ueG{Z&O@g@A;c&Y440-n8OJ_c7sq}q~X;n z@a*~_2?y&TqoelQ*4jMeHv4YyuE0Vg6SXors$wzE%GUmrF4Q$%|3-%S#wQ)NuI9E7 z%KEbRiZwRzzfNG^Z%o?_5*`tnz-Ul}h~NK1lz|2~ z-0A21Ah7f6oR{}-PqF57FBbTiz?x}e?#AKX-!{aLMIB5&@B1y?CH2X)^@e0GCME_lTq&ygndp!?SB z*Z0BF*WF8*xnfw^rTiTHPSmx;4|6E}qPWNk_5Q+%bsY!K4gLMAOAIH4)cL}{Q`M(7 z7(82V903>0pE)=7PPa@a(BU>_TGH;)z_~bc_D;u~n=BUd_-np7SCT@gmp*O(4EMeK zRqAIK;U2>JN<@63(~tkDee37y?Sq|tSHO_H=}3Vsoj$#(JPPxyA)%vf7yfr{a+g(% zVvb~+ZmyIC*GGlA=U~nVwVGr<0!LalZPmmaqP!DBW5`YI$@@blm=kQS-|_s1b8#Q- V%oQdMz|Ph(u5#Y_IQE;J?;jiblj;Bf literal 0 HcmV?d00001 diff --git a/contribs/simwrapper/src/test/resources/kehlheim_test.shx b/contribs/simwrapper/src/test/resources/kehlheim_test.shx new file mode 100644 index 0000000000000000000000000000000000000000..9612c3d6be5bdfd876a38f5a7b7eb4dac6e15846 GIT binary patch literal 164 zcmZQzQ0HR64uW1VGcd3MNurmEwAsI5Xbuqj4b}&RCPQaE6$y7xd{syEg0Qt`tv;Y7A literal 0 HcmV?d00001