Feat: download Toggl detailed time reports as CSV (#26)

.env.sample

@@ -4,3 +4,9 @@ HARVEST_DATA=data/harvest-sample.csv
 TOGGL_DATA=data/toggl-sample.csv
 TOGGL_PROJECT_INFO=data/toggl-project-info-sample.json
 TOGGL_USER_INFO=data/toggl-user-info-sample.json
+
+TOGGL_API_TOKEN=token
+TOGGL_CLIENT_ID=client
+TOGGL_WORKSPACE_ID=workspace
+
+TZ_NAME=America/Los_Angeles

README.md

@@ -62,14 +62,35 @@ The `time` command provides an interface for working with time entries from Comp
 
 ```bash
 $ compiler-admin time -h
-usage: compiler-admin time [-h] {convert} ...
+usage: compiler-admin time [-h] {convert,download} ...
 
 positional arguments:
-  {convert}   The time command to run.
-    convert   Convert a time report from one format into another.
+  {convert,download}  The time command to run.
+    convert           Convert a time report from one format into another.
+    download          Download a Toggl report in CSV format.
 
 options:
-  -h, --help  show this help message and exit
+  -h, --help          show this help message and exit
+```
+
+### Downloading a Toggl report
+
+Use this command to download a time report from Toggl in CSV format:
+
+```bash
+$ compiler-admin time download -h
+usage: compiler-admin time download [-h] [--start YYYY-MM-DD] [--end YYYY-MM-DD] [--output OUTPUT]
+                                    [--client CLIENT_ID] [--project PROJECT_ID] [--task TASK_ID] [--user USER_ID]
+
+options:
+  -h, --help            show this help message and exit
+  --start YYYY-MM-DD    The start date of the reporting period. Defaults to the beginning of the prior month.
+  --end YYYY-MM-DD      The end date of the reporting period. Defaults to the end of the prior month.
+  --output OUTPUT       The path to the file where converted data should be written. Defaults to stdout.
+  --client CLIENT_ID    An ID for a Toggl Client to filter for in reports. Can be supplied more than once.
+  --project PROJECT_ID  An ID for a Toggl Project to filter for in reports. Can be supplied more than once.
+  --task TASK_ID        An ID for a Toggl Project Task to filter for in reports. Can be supplied more than once.
+  --user USER_ID        An ID for a Toggl User to filter for in reports. Can be supplied more than once.
 ```
 
 ### Converting an hours report

compiler_admin/commands/time/__init__.py

@@ -1,6 +1,7 @@
 from argparse import Namespace
 
 from compiler_admin.commands.time.convert import convert  # noqa: F401
+from compiler_admin.commands.time.download import download  # noqa: F401
 
 
 def time(args: Namespace, *extra):

compiler_admin/commands/time/download.py

@@ -0,0 +1,21 @@
+from argparse import Namespace
+
+from compiler_admin import RESULT_SUCCESS
+from compiler_admin.services.toggl import INPUT_COLUMNS as TOGGL_COLUMNS, download_time_entries
+
+
+def download(args: Namespace, *extras):
+    params = dict(start_date=args.start, end_date=args.end, output_path=args.output, output_cols=TOGGL_COLUMNS)
+
+    if args.client_ids:
+        params.update(dict(client_ids=args.client_ids))
+    if args.project_ids:
+        params.update(dict(project_ids=args.project_ids))
+    if args.task_ids:
+        params.update(dict(task_ids=args.task_ids))
+    if args.user_ids:
+        params.update(dict(user_ids=args.user_ids))
+
+    download_time_entries(**params)
+
+    return RESULT_SUCCESS

compiler_admin/main.py

@@ -1,6 +1,10 @@
 from argparse import ArgumentParser, _SubParsersAction
+from datetime import datetime, timedelta
+import os
 import sys
 
+from pytz import timezone
+
 from compiler_admin import __version__ as version
 from compiler_admin.commands.info import info
 from compiler_admin.commands.init import init
@@ -9,6 +13,24 @@
 from compiler_admin.commands.user.convert import ACCOUNT_TYPE_OU
 
 
+TZINFO = timezone(os.environ.get("TZ_NAME", "America/Los_Angeles"))
+
+
+def local_now():
+    return datetime.now(tz=TZINFO)
+
+
+def prior_month_end():
+    now = local_now()
+    first = now.replace(day=1)
+    return first - timedelta(days=1)
+
+
+def prior_month_start():
+    end = prior_month_end()
+    return end.replace(day=1)
+
+
 def add_sub_cmd_parser(parser: ArgumentParser, dest="subcommand", help=None):
     """Helper adds a subparser for the given dest."""
     return parser.add_subparsers(dest=dest, help=help)
@@ -54,6 +76,57 @@ def setup_time_command(cmd_parsers: _SubParsersAction):
     )
     time_convert.add_argument("--client", default=None, help="The name of the client to use in converted data.")
 
+    time_download = add_sub_cmd(time_subcmds, "download", help="Download a Toggl report in CSV format.")
+    time_download.add_argument(
+        "--start",
+        metavar="YYYY-MM-DD",
+        default=prior_month_start(),
+        type=lambda s: TZINFO.localize(datetime.strptime(s, "%Y-%m-%d")),
+        help="The start date of the reporting period. Defaults to the beginning of the prior month.",
+    )
+    time_download.add_argument(
+        "--end",
+        metavar="YYYY-MM-DD",
+        default=prior_month_end(),
+        type=lambda s: TZINFO.localize(datetime.strptime(s, "%Y-%m-%d")),
+        help="The end date of the reporting period. Defaults to the end of the prior month.",
+    )
+    time_download.add_argument(
+        "--output", default=sys.stdout, help="The path to the file where converted data should be written. Defaults to stdout."
+    )
+    time_download.add_argument(
+        "--client",
+        dest="client_ids",
+        metavar="CLIENT_ID",
+        action="append",
+        type=int,
+        help="An ID for a Toggl Client to filter for in reports. Can be supplied more than once.",
+    )
+    time_download.add_argument(
+        "--project",
+        dest="project_ids",
+        metavar="PROJECT_ID",
+        action="append",
+        type=int,
+        help="An ID for a Toggl Project to filter for in reports. Can be supplied more than once.",
+    )
+    time_download.add_argument(
+        "--task",
+        dest="task_ids",
+        metavar="TASK_ID",
+        action="append",
+        type=int,
+        help="An ID for a Toggl Project Task to filter for in reports. Can be supplied more than once.",
+    )
+    time_download.add_argument(
+        "--user",
+        dest="user_ids",
+        metavar="USER_ID",
+        action="append",
+        type=int,
+        help="An ID for a Toggl User to filter for in reports. Can be supplied more than once.",
+    )
+
 
 def setup_user_command(cmd_parsers: _SubParsersAction):
     user_cmd = add_sub_cmd(cmd_parsers, "user", help="Work with users in the Compiler org.")

compiler_admin/services/toggl.py

@@ -1,12 +1,22 @@
+from base64 import b64encode
+from datetime import datetime
+import io
 import os
 import sys
 from typing import TextIO
 
 import pandas as pd
+import requests
 
+from compiler_admin import __version__
 from compiler_admin.services.google import user_info as google_user_info
 import compiler_admin.services.files as files
 
+# Toggl API config
+API_BASE_URL = "https://api.track.toggl.com"
+API_REPORTS_BASE_URL = "reports/api/v3"
+API_WORKSPACE = "workspace/{}"
+
 # cache of previously seen project information, keyed on Toggl project name
 PROJECT_INFO = {}
 
@@ -36,6 +46,50 @@ def _get_info(obj: dict, key: str, env_key: str):
     return obj.get(key)
 
 
+def _toggl_api_authorization_header():
+    """Gets an `Authorization: Basic xyz` header using the Toggl API token.
+
+    See https://engineering.toggl.com/docs/authentication.
+    """
+    token = _toggl_api_token()
+    creds = f"{token}:api_token"
+    creds64 = b64encode(bytes(creds, "utf-8")).decode("utf-8")
+    return {"Authorization": "Basic {}".format(creds64)}
+
+
+def _toggl_api_headers():
+    """Gets a dict of headers for Toggl API requests.
+
+    See https://engineering.toggl.com/docs/.
+    """
+    headers = {"Content-Type": "application/json"}
+    headers.update({"User-Agent": "compilerla/compiler-admin:{}".format(__version__)})
+    headers.update(_toggl_api_authorization_header())
+    return headers
+
+
+def _toggl_api_report_url(endpoint: str):
+    """Get a fully formed URL for the Toggl Reports API v3 endpoint.
+
+    See https://engineering.toggl.com/docs/reports_start.
+    """
+    workspace_id = _toggl_workspace()
+    return "/".join((API_BASE_URL, API_REPORTS_BASE_URL, API_WORKSPACE.format(workspace_id), endpoint))
+
+
+def _toggl_api_token():
+    """Gets the value of the TOGGL_API_TOKEN env var."""
+    return os.environ.get("TOGGL_API_TOKEN")
+
+
+def _toggl_client_id():
+    """Gets the value of the TOGGL_CLIENT_ID env var."""
+    client_id = os.environ.get("TOGGL_CLIENT_ID")
+    if client_id:
+        return int(client_id)
+    return None
+
+
 def _toggl_project_info(project: str):
     """Return the cached project for the given project key."""
     return _get_info(PROJECT_INFO, project, "TOGGL_PROJECT_INFO")
@@ -46,6 +100,11 @@ def _toggl_user_info(email: str):
     return _get_info(USER_INFO, email, "TOGGL_USER_INFO")
 
 
+def _toggl_workspace():
+    """Gets the value of the TOGGL_WORKSPACE_ID env var."""
+    return os.environ.get("TOGGL_WORKSPACE_ID")
+
+
 def _get_first_name(email: str) -> str:
     """Get cached first name or derive from email."""
     user = _toggl_user_info(email)
@@ -127,3 +186,75 @@ def convert_to_harvest(
     source["Hours"] = (source["Duration"].dt.total_seconds() / 3600).round(2)
 
     files.write_csv(output_path, source, columns=output_cols)
+
+
+def download_time_entries(
+    start_date: datetime,
+    end_date: datetime,
+    output_path: str | TextIO = sys.stdout,
+    output_cols: list[str] | None = INPUT_COLUMNS,
+    **kwargs,
+):
+    """Download a CSV report from Toggl of detailed time entries for the given date range.
+
+    Args:
+        start_date (datetime): The beginning of the reporting period.
+
+        end_date (str): The end of the reporting period.
+
+        output_path: The path to a CSV file where Toggl time entries will be written; or a writeable buffer for the same.
+
+        output_cols (list[str]): A list of column names for the output.
+
+    Extra kwargs are passed along in the POST request body.
+
+    By default, requests a report with the following configuration:
+        * `billable=True`
+        * `client_ids=[$TOGGL_CLIENT_ID]`
+        * `rounding=1` (True, but this is an int param)
+        * `rounding_minutes=15`
+
+    See https://engineering.toggl.com/docs/reports/detailed_reports#post-export-detailed-report.
+
+    Returns:
+        None. Either prints the resulting CSV data or writes to output_path.
+    """
+    start = start_date.strftime("%Y-%m-%d")
+    end = end_date.strftime("%Y-%m-%d")
+    # calculate a timeout based on the size of the reporting period in days
+    # approximately 5 seconds per month of query size, with a minimum of 5 seconds
+    range_days = (end_date - start_date).days
+    timeout = int((max(30, range_days) / 30.0) * 5)
+
+    if ("client_ids" not in kwargs or not kwargs["client_ids"]) and isinstance(_toggl_client_id(), int):
+        kwargs["client_ids"] = [_toggl_client_id()]
+
+    params = dict(
+        billable=True,
+        start_date=start,
+        end_date=end,
+        rounding=1,
+        rounding_minutes=15,
+    )
+    params.update(kwargs)
+
+    headers = _toggl_api_headers()
+    url = _toggl_api_report_url("search/time_entries.csv")
+
+    response = requests.post(url, json=params, headers=headers, timeout=timeout)
+    response.raise_for_status()
+
+    # the raw response has these initial 3 bytes:
+    #
+    #   b"\xef\xbb\xbfUser,Email,Client..."
+    #
+    # \xef\xbb\xb is the Byte Order Mark (BOM) sometimes used in unicode text files
+    # these 3 bytes indicate a utf-8 encoded text file
+    #
+    # See more
+    #  - https://en.wikipedia.org/wiki/Byte_order_mark
+    #  - https://stackoverflow.com/a/50131187
+    csv = response.content.decode("utf-8-sig")
+
+    df = pd.read_csv(io.StringIO(csv))
+    files.write_csv(output_path, df, columns=output_cols)