Merged branch develop into master

global.R

@@ -8,6 +8,8 @@ library(leaflet) # For now : devtools::install_github("RCura/leaflet")
 library(ggthemes)
 library(ggmap)
 
+options(shiny.maxRequestSize = 8*1024^2)
+
 moisFr <- c(
   "janv.",
   "févr.",
@@ -32,9 +34,10 @@ colorPalette <- colorNumeric(
   )
 
 
-formatData <- function(rawData){
-  rawData %>%
+formatData <- function(rawData, tz){
+  formattedData <- rawData %>%
     # mutate(time = parse_datetime(Time, format = "%Y/%m/%d %H:%M:%S")) %>%
+    mutate(Time = with_tz(Time, tzone = tz)) %>%
     mutate(jour = day(Time)) %>%
     mutate(jourN = factor(weekdays(Time, abbreviate = TRUE), levels = joursFr)) %>%
     mutate(mois = month(Time)) %>%
@@ -43,6 +46,10 @@ formatData <- function(rawData){
     mutate(heure = hour(Time)) %>%
     mutate(minute = minute(Time)) %>%
     mutate(dhour = hour(Time) + minute(Time) / 60 + second(Time) / 3600)
+  if (nrow(formattedData) > 50E3){
+    formattedData <- formattedData %>%
+      sample_n(size = 50E3)
+  }
 }
 
 theme_timelineEDB <- function() {
@@ -73,9 +80,44 @@ theme_timelineEDB <- function() {
 
 
 rawData <- read_csv(file = "data/SelfPoints.csv") %>%
-  rename(Time = timestamp) %>%
   select(Time, X, Y)
 
-formattedData <- formatData(rawData)
+formattedData <- formatData(rawData,  "Europe/Paris")
+
+
+ZipPath <- "data/takeout-20160629T102959Z.zip"
+google_jsonZip_to_DF <- function(ZipPath, tz){
+
+  # Extract JSON from ZIP
+  ## Detecting files inside ZIP
+  zipFiles <- unzip(ZipPath, list = TRUE)
+  jsonPath <- zipFiles[grepl(zipFiles$Name,pattern = ".json"),]
+  ## Unzipping
+  unzip(ZipPath, files = jsonPath$Name,
+        overwrite = TRUE,
+        junkpaths = TRUE,
+        exdir = tempdir())
+  extractedFile <- paste(tempdir(),basename(jsonPath$Name), sep = "/")
+  # Convert JSON to CSV
+  jsonFile <- tempfile(fileext = ".json")
+  ## Make sure not conflicting
+  file.rename(from = extractedFile, to = jsonFile)
+  csvFile <- tempfile(fileext = ".csv")
+  ## Python call
+  cmdCall <- sprintf("python %s %s --output %s --format csv",
+                     "src/location_history_json_converter.py",
+                     jsonFile,
+                     csvFile)
+  system(cmdCall)
+  ## Clean
+  file.remove(jsonFile)
+
+  # Read CSV
+  resultDF <- read_csv(csvFile) %>%
+    separate(Location, into = c("Y", "X"),  sep = " ",  remove = TRUE, convert = TRUE)
+
+  # Format it correctly
+  formatData(resultDF, tz)  
+}
 
 

server.R

@@ -3,15 +3,19 @@ library(shiny)
 shinyServer(function(session, input, output) {
   locationData <-
     reactiveValues(
-      raw = rawData,
       base = formattedData,
       geofiltred = NA,
       timefiltred = NA
     )
   analysisData <- reactiveValues(homePoint = NA, workPoint = NA)
 
-
-
+  observe({
+    req(input$userData)
+    locationData$base <- google_jsonZip_to_DF(input$userData$datapath, input$timezone)
+    locationData$geofiltred <- NA
+    locationData$timefiltred <- NA
+  })
+
   output$map <- renderLeaflet({
     mapData <- locationData$base
     dataLength <- nrow(mapData)

src/location_history_json_converter.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Gerwin Sturm, FoldedSoft e.U. / www.foldedsoft.at
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import division
+
+import sys
+import json
+import math
+from argparse import ArgumentParser
+from datetime import datetime
+
+
+def main(argv):
+    arg_parser = ArgumentParser()
+    arg_parser.add_argument("input", help="Input File (JSON)")
+    arg_parser.add_argument("-o", "--output", help="Output File (will be overwritten!)")
+    arg_parser.add_argument("-f", "--format", choices=["kml", "json", "csv", "js", "gpx", "gpxtracks"], default="kml", help="Format of the output")
+    arg_parser.add_argument("-v", "--variable", default="locationJsonData", help="Variable name to be used for js output")
+    args = arg_parser.parse_args()
+    if not args.output: #if the output file is not specified, set to input filename with a diffrent extension
+        args.output = '.'.join(args.input.split('.')[:-1]) + '.'+args.format
+    if args.input == args.output:
+        arg_parser.error("Input and output have to be different files")
+        return
+
+    try:
+        json_data = open(args.input).read()
+    except:
+        print("Error opening input file")
+        return
+
+    try:
+        data = json.loads(json_data)
+    except:
+        print("Error decoding json")
+        return
+
+    if "locations" in data and len(data["locations"]) > 0:
+        try:
+            f_out = open(args.output, "w")
+        except:
+            print("Error creating output file for writing")
+            return
+
+        items = data["locations"]
+
+        if args.format == "json" or args.format == "js":
+            if args.format == "js":
+                f_out.write("window.%s = " % args.variable)
+
+            f_out.write("{\n")
+            f_out.write("  \"data\": {\n")
+            f_out.write("    \"items\": [\n")
+            first = True
+
+            for item in items:
+                if first:
+                    first = False
+                else:
+                    f_out.write(",\n")
+                f_out.write("      {\n")
+                f_out.write("         \"timestampMs\": %s,\n" % item["timestampMs"])
+                f_out.write("         \"latitude\": %s,\n" % (item["latitudeE7"] / 10000000))
+                f_out.write("         \"longitude\": %s\n" % (item["longitudeE7"] / 10000000))
+                f_out.write("      }")
+            f_out.write("\n    ]\n")
+            f_out.write("  }\n}")
+            if args.format == "js":
+                f_out.write(";")
+
+        if args.format == "csv":
+            f_out.write("Time,Location\n")
+            for item in items:
+                f_out.write(datetime.fromtimestamp(int(item["timestampMs"]) / 1000).strftime("%Y-%m-%d %H:%M:%S"))
+                f_out.write(",")
+                f_out.write("%s %s\n" % (item["latitudeE7"] / 10000000, item["longitudeE7"] / 10000000))
+
+        if args.format == "kml":
+            f_out.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
+            f_out.write("<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n")
+            f_out.write("  <Document>\n")
+            f_out.write("    <name>Location History</name>\n")
+            for item in items:
+                f_out.write("    <Placemark>\n")
+                # Order of these tags is important to make valid KML: TimeStamp, ExtendedData, then Point
+                f_out.write("      <TimeStamp><when>")
+                f_out.write(datetime.fromtimestamp(int(item["timestampMs"]) / 1000).strftime("%Y-%m-%dT%H:%M:%SZ"))
+                f_out.write("</when></TimeStamp>\n")
+                if "accuracy" in item or "speed" in item or "altitude" in item:
+                    f_out.write("      <ExtendedData>\n")
+                    if "accuracy" in item:
+                        f_out.write("        <Data name=\"accuracy\">\n")
+                        f_out.write("          <value>%d</value>\n" % item["accuracy"])
+                        f_out.write("        </Data>\n")
+                    if "speed" in item:
+                        f_out.write("        <Data name=\"speed\">\n")
+                        f_out.write("          <value>%d</value>\n" % item["speed"])
+                        f_out.write("        </Data>\n")
+                    if "altitude" in item:
+                        f_out.write("        <Data name=\"altitude\">\n")
+                        f_out.write("          <value>%d</value>\n" % item["altitude"])
+                        f_out.write("        </Data>\n")
+                    f_out.write("      </ExtendedData>\n")
+                f_out.write("      <Point><coordinates>%s,%s</coordinates></Point>\n" % (item["longitudeE7"] / 10000000, item["latitudeE7"] / 10000000))
+                f_out.write("    </Placemark>\n")
+            f_out.write("  </Document>\n</kml>\n")
+
+        if args.format == "gpx" or args.format == "gpxtracks":
+            f_out.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
+            f_out.write("<gpx xmlns=\"http://www.topografix.com/GPX/1/1\" version=\"1.1\" creator=\"Google Latitude JSON Converter\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd\">\n")
+            f_out.write("  <metadata>\n")
+            f_out.write("    <name>Location History</name>\n")
+            f_out.write("  </metadata>\n")
+            if args.format == "gpx":
+                for item in items:
+                    f_out.write("  <wpt lat=\"%s\" lon=\"%s\">\n"  % (item["latitudeE7"] / 10000000, item["longitudeE7"] / 10000000))
+                    if "altitude" in item:
+                        f_out.write("    <ele>%d</ele>\n" % item["altitude"])
+                    f_out.write("    <time>%s</time>\n" % str(datetime.fromtimestamp(int(item["timestampMs"]) / 1000).strftime("%Y-%m-%dT%H:%M:%SZ")))
+                    f_out.write("    <desc>%s" % datetime.fromtimestamp(int(item["timestampMs"]) / 1000).strftime("%Y-%m-%d %H:%M:%S"))
+                    if "accuracy" in item or "speed" in item:
+                        f_out.write(" (")
+                        if "accuracy" in item:
+                            f_out.write("Accuracy: %d" % item["accuracy"])
+                        if "accuracy" in item and "speed" in item:
+                            f_out.write(", ")
+                        if "speed" in item:
+                            f_out.write("Speed:%d" % item["speed"])
+                        f_out.write(")")
+                    f_out.write("</desc>\n")
+                    f_out.write("  </wpt>\n")
+            if args.format == "gpxtracks":
+                f_out.write("  <trk>\n")
+                f_out.write("    <trkseg>\n")
+                lastloc = None
+                # The deltas below assume input is in reverse chronological order.  If it's not, uncomment this:
+                # items = sorted(data["data"]["items"], key=lambda x: x['timestampMs'], reverse=True)
+                for item in items:
+                    if lastloc:
+                        timedelta = -((int(item['timestampMs']) - int(lastloc['timestampMs'])) / 1000 / 60)
+                        distancedelta = getDistanceFromLatLonInKm(item['latitudeE7'] / 10000000, item['longitudeE7'] / 10000000, lastloc['latitudeE7'] / 10000000, lastloc['longitudeE7'] / 10000000)
+                        if timedelta > 10 or distancedelta > 40:
+                            # No points for 10 minutes or 40km in under 10m? Start a new track.
+                            f_out.write("    </trkseg>\n")
+                            f_out.write("  </trk>\n")
+                            f_out.write("  <trk>\n")
+                            f_out.write("    <trkseg>\n")
+                    f_out.write("      <trkpt lat=\"%s\" lon=\"%s\">\n" % (item["latitudeE7"] / 10000000, item["longitudeE7"] / 10000000))
+                    if "altitude" in item:
+                        f_out.write("        <ele>%d</ele>\n" % item["altitude"])
+                    f_out.write("        <time>%s</time>\n" % str(datetime.fromtimestamp(int(item["timestampMs"]) / 1000).strftime("%Y-%m-%dT%H:%M:%SZ")))
+                    if "accuracy" in item or "speed" in item:
+                        f_out.write("        <desc>\n")
+                        if "accuracy" in item:
+                            f_out.write("          Accuracy: %d\n" % item["accuracy"])
+                        if "speed" in item:
+                            f_out.write("          Speed:%d\n" % item["speed"])
+                        f_out.write("        </desc>\n")
+                    f_out.write("      </trkpt>\n")
+                    lastloc = item
+                f_out.write("    </trkseg>\n")
+                f_out.write("  </trk>\n")
+            f_out.write("</gpx>\n")
+
+        f_out.close()
+
+    else:
+        print("No data found in json")
+        return
+
+
+# Haversine formula
+def getDistanceFromLatLonInKm(lat1,lon1,lat2,lon2):
+    R = 6371 # Radius of the earth in km
+    dlat = deg2rad(lat2-lat1)
+    dlon = deg2rad(lon2-lon1)
+    a = math.sin(dlat/2) * math.sin(dlat/2) + \
+    math.cos(deg2rad(lat1)) * math.cos(deg2rad(lat2)) * \
+    math.sin(dlon/2) * math.sin(dlon/2)
+    c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
+    d = R * c # Distance in km
+    return d
+
+
+def deg2rad(deg):
+    return deg * (math.pi/180)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))