Skip to content

Commit

Permalink
setup.py structure and some PEP-8 style formatting #2
Browse files Browse the repository at this point in the history
  • Loading branch information
hcwinsemius committed Jun 18, 2021
1 parent b4e05ed commit f9e44f5
Show file tree
Hide file tree
Showing 10 changed files with 95 additions and 51 deletions.
25 changes: 24 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,27 @@
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg

__pycache__
credentials.txt
example_files
venv/
odk2odm_venv/

# PyCharm
.idea/

7 changes: 7 additions & 0 deletions odk2odm/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
from odk2odm import attachments
from odk2odm import csv_from_odata
from odk2odm import exif_locate
from odk2odm import extract_location_from_exif
from odk2odm import fetch
from odk2odm import image_integrity_check
from odk2odm import make_geo_txt
11 changes: 6 additions & 5 deletions attachments.py → odk2odm/attachments.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
#!/usr/bin/python3

import sys, os
import os
import fetch
import argparse
import threading
import csv


def threaded_download():
"""Grab lots of photos using threaded concurrent download"""
# NOT DONE
# FIXME: NOT DONE

threads = []

Expand All @@ -21,6 +21,7 @@ def threaded_download():
for thread in threads:
thread.join()


def all_attachments_from_form(url, aut, project, form, outdir):
"""Downloads all available attachments from a given form"""
submissions = fetch.submissions(url, aut, project, form)
Expand All @@ -32,11 +33,12 @@ def all_attachments_from_form(url, aut, project, form, outdir):
for attachment in attachments.json():
fn = attachment['name']
attresp = fetch.attachment(url, aut, project, form,
sub_id, fn)
sub_id, fn)
outfilepath = os.path.join(outdir, fn)
with open(outfilepath, 'wb') as outfile:
outfile.write(attresp.content)


def specified_attachments_from_form(url, aut, project, form, outdir, infile):
"""Downloads attachments specified in a text file."""
wantedfiles = []
Expand Down Expand Up @@ -72,4 +74,3 @@ def specified_attachments_from_form(url, aut, project, form, outdir, infile):

all_attachments_from_form(args.base_url, (args.user, args.password),
args.project, args.form, args.output_directory)

7 changes: 5 additions & 2 deletions csv_from_odata.py → odk2odm/csv_from_odata.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
#!/usr/bin/python3

import sys, os
import os
import fetch
import argparse
import csv
import json


def csv_from_odata(url, aut, project, form, outdir, gc):
"""Write a CSV to a specified directory using odata for a specified form"""
response = fetch.odata_submissions(url, aut, project, form)
Expand All @@ -29,6 +30,7 @@ def csv_from_odata(url, aut, project, form, outdir, gc):
geolist = jsonpoint_to_list(gc_contents)
w.writerow(row[: geocol] + geolist + row[geocol :])


def jsonpoint_to_list(po):
"""ODK Central returnt point in what is almost a JSON string,
except that it is single-quoted instead of double-quoted,
Expand All @@ -55,7 +57,8 @@ def jsonpoint_to_list(po):
except Exception as f:
#print(f)
return ['', '', '', '']



if __name__ == '__main__':
p = argparse.ArgumentParser()
p.add_argument('-url', '--base_url',
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import csv
import exifread


def scandir(dir):
"""Walk recursively through a directory and return a list of all files in it"""
filelist = []
Expand All @@ -31,6 +32,7 @@ def scandir(dir):
filelist.append(os.path.join(path, f))
return filelist


def exif_GPS_to_decimal_degrees(intag):
"""
Spit out a decimal degree lat or long.
Expand All @@ -41,10 +43,12 @@ def exif_GPS_to_decimal_degrees(intag):
s = float(intag.values[2].num) / float(intag.values[2].den)
return d + (m / 60.0) + (s / 3600.0)


def exif_GPS_alt_to_decimal_m(intag):
alt = intag.values[0]
return alt.decimal()



def extract_location(infile):
"""Return the GPS lat and long of a photo from EXIF in decimal degrees"""
with open(infile, 'rb') as f:
Expand All @@ -69,8 +73,8 @@ def extract_location(infile):
except Exception as e:
print(e)
print('The photo {} failed for some reason'.format(infile))


def create_geotag_list(indir):
"""Create a CSV file with a list of photos and their lat & long"""
outfile = indir + '.csv'
Expand All @@ -87,8 +91,8 @@ def create_geotag_list(indir):
if(crds):
writer.writerow([image_filename, image_file, image_dirname,
crds[0], crds[1], crds[2]])


if __name__ == "__main__":
"""Expects a directory as the sole argument"""
create_geotag_list(sys.argv[1])
14 changes: 8 additions & 6 deletions fetch.py → odk2odm/fetch.py
Original file line number Diff line number Diff line change
@@ -1,29 +1,32 @@
#!/usr/bin/python3

import sys, os
import requests


def projects(base_url, aut):
"""Fetch a list of projects on an ODK Central server."""
url = f'{base_url}/v1/projects'
return requests.get(url, auth = aut)


def forms(base_url, aut, projectId):
"""Fetch a list of forms in a project."""
url = f'{base_url}/v1/projects/{projectId}/forms'
return requests.get(url, auth = aut)


def submissions(base_url, aut, projectId, formId):
"""Fetch a list of submission instances for a given form."""
url = f'{base_url}/v1/projects/{projectId}/forms/{formId}/submissions'
return requests.get(url, auth = aut)


# Should work with ?media=false appended but doesn't. Use the odata version.
def csv_submissions(base_url, aut, projectId, formId):
"""Fetch a CSV file of the submissions to a survey form."""
f'{base_url}/v1/projects/{projectId}/forms/{formId}/submissions.csv.zip'
return requests.get(url, auth = aut)


def odata_submissions(base_url, aut, projectId, formId):
"""
Fetch the submissions using the odata api.
Expand All @@ -34,21 +37,20 @@ def odata_submissions(base_url, aut, projectId, formId):
submissions = requests.get(url, auth = aut)
return submissions


def attachment_list(base_url, aut, projectId, formId, instanceId):
"""Fetch an individual media file attachment."""
url = f'{base_url}/v1/projects/{projectId}/forms/{formId}/submissions/'\
f'{instanceId}/attachments'
return requests.get(url, auth = aut)


def attachment(base_url, aut, projectId, formId, instanceId, filename):
"""Fetch a specific attachment by filename from a submission to a form."""
url = f'{base_url}/v1/projects/{projectId}/forms/{formId}/submissions/'\
f'{instanceId}/attachments/{filename}'
return requests.get(url, auth = aut)


if __name__ == '__main__':
pass




7 changes: 4 additions & 3 deletions image_integrity_check.py → odk2odm/image_integrity_check.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
#!/usr/bin/python3

import sys, os
import os
import sys
from pathlib import Path
from PIL import Image


def checkimages(indir):
"""returns a list of image files that can be successfully opened by PIL"""
filelist = []
Expand All @@ -20,8 +21,8 @@ def checkimages(indir):
except Exception as e:
print(e)
badfiles.append(f)
return goodfiles, badfiles

return(goodfiles, badfiles)

def write_file_lists(indir):
"""Write two text files with the good and bad image files in the parent
Expand Down
48 changes: 27 additions & 21 deletions make_geo_txt.py → odk2odm/make_geo_txt.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,13 @@
care).
"""
################################################################################
import sys, os
import os
import csv
import re
import argparse
import string



def make_geo_txt(infile, colrange, lonc,
latc, elec, accc, proj):
"""
Expand All @@ -36,11 +36,11 @@ def make_geo_txt(infile, colrange, lonc,
loncol = col2num(lonc)
elecol = col2num(elec)
acccol = col2num(accc)
sites = list(csv.reader(open(infile), delimiter = ';'))[1:]
sites = list(csv.reader(open(infile), delimiter=';'))[1:]
cols = parse_range(colrange)
outfile = os.path.join(os.path.dirname(infile), 'geo.txt')
with (open(outfile, 'w')) as csvfile:
w = csv.writer(csvfile, delimiter = ' ')
w = csv.writer(csvfile, delimiter=' ')
w.writerow([proj])
for site in sites:
for col in cols:
Expand All @@ -50,15 +50,16 @@ def make_geo_txt(infile, colrange, lonc,
site[int(loncol) - 1],
site[int(latcol) - 1],
site[int(elecol) - 1],
'0','0','0',
'0', '0', '0',
site[int(acccol) - 1],
site[int(acccol) - 1],
])
except Exception as e:
print(site)
print(col)
print(e)



def col2num(col):
"""Excel column letters to 1-based column number"""
if col.isdigit():
Expand All @@ -71,6 +72,7 @@ def col2num(col):
ord('A')) + 1
return colnum


def parse_range(instring):
"""
Takes a string like '3-5, 13, 36-38'
Expand All @@ -84,43 +86,47 @@ def parse_range(instring):
l = ([range(int(i[0]), int(i[1]) + 1)
if len(i) == 2
else i[0] for i in numparts])

return [int(item) for sublist in l for item in sublist]


if __name__ == "__main__":
"""
"""
p = argparse.ArgumentParser(description =
p = argparse.ArgumentParser(description=
('Georeference attachments '
'from ODK submissions using '
'lat and lon columns.'))
p.add_argument('inputfile', help = 'Input CSV file')
p.add_argument('inputfile', help='Input CSV file')
p.add_argument('-r', '--range', required=True,
help = ('columns with attachments to be '
help=('columns with attachments to be '
'georeferenced. Use format '
'"3-5,7,9-11" or "c-e,g,i-k" '
'(spreadsheet format). Use 1-based '
'column numbers'))
p.add_argument('-lat', '--latitude',
help = ('Latitude column. Can be 1-based '
help=('Latitude column. Can be 1-based '
'column number or spreadsheet '
'column letters'))
p.add_argument('-lon', '--longitude',
help = ('longitude column '
help=('longitude column '
'can be 1-based column number or '
'spreadsheet column letters'))
p.add_argument('-ele', '--elevation',
help = 'GPS elevation column')
help='GPS elevation column')
p.add_argument('-acc', '--accuracy',
help = 'Estimated GPS accuracy column')
help='Estimated GPS accuracy column')
p.add_argument('-proj', '--projection',
help = 'Coordinate Reference System',
default = 'EPSG:4326')
help='Coordinate Reference System',
default='EPSG:4326')
args = p.parse_args()

make_geo_txt(args.inputfile, args.range,
args.longitude, args.latitude,
args.elevation, args.accuracy,
args.projection)
make_geo_txt(
args.inputfile,
args.range,
args.longitude,
args.latitude,
args.elevation,
args.accuracy,
args.projection
)
13 changes: 5 additions & 8 deletions script/setup.sh → setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,19 @@ if [ -d venv ]
then
echo venv was already present, hope it is the right one
else
python3 -m venv venv
python3 -m venv odk2odm_venv
fi

echo creating and activating virtual environmnet
pip3 install virtualenv
source venv/bin/activate
source odk2odm_venv/bin/activate

echo installing requests
pip install requests

echo installing exifread
pip install exifread
echo installing odk2odm library and dependencies
pip install .

echo Done.
echo To use the utilities here, activate the virtual environment with:
echo source venv/bin/activate
echo source odk2odm_venv/bin/activate
echo
echo And type deactivate to get out when you are done.

Expand Down

0 comments on commit f9e44f5

Please sign in to comment.