-
Notifications
You must be signed in to change notification settings - Fork 14
/
container_init.sh
executable file
·71 lines (52 loc) · 5.1 KB
/
container_init.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
#!/bin/bash
echo
echo " ====="
echo "| starting backend container"
echo " ====="
echo
# Download WMG data snapshot to a mounted filesystem of the compute node on AWS
# This is done as optimization because retrieving data from local disk is
# significantly faster than retrieving data from S3
WMG_SNAPSHOT_FS_CACHE_ROOT_PATH="/single-cell-data-portal/census_cube_snapshot_cache"
# LATEST_READER_SNAPSHOT_SCHEMA_VERSION here and WMG_API_SNAPSHOT_SCHEMA_VERSION
# in backend/common/census_cube/api/config.py should have the same value
LATEST_READER_SNAPSHOT_SCHEMA_VERSION="v5"
echo "| ENV VAR DOWNLOAD_WMG_DATA_TO_DISK: ${DOWNLOAD_WMG_DATA_TO_DISK}"
if [[ "${DOWNLOAD_WMG_DATA_TO_DISK}" == "false" ]]; then
echo "| Skipping downloading WMG data snapshot because DOWNLOAD_WMG_DATA_TO_DISK is set to false"
elif [[ "${DEPLOYMENT_STAGE}" == "rdev" && -n "${REMOTE_DEV_PREFIX}" ]]; then
echo "| Downloading WMG data snapshot for RDEV stack: ${REMOTE_DEV_PREFIX} from S3 to filesystem path: ${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH}"
strip_slash_remote_dev_prefix="${REMOTE_DEV_PREFIX//\//}" # strips ALL "/"
latest_snapshot_identifier=$(aws s3 cp "s3://env-rdev-wmg/${strip_slash_remote_dev_prefix}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/latest_snapshot_identifier" -)
echo aws s3 sync "s3://env-rdev-wmg/${strip_slash_remote_dev_prefix}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/${latest_snapshot_identifier}" "${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH}/${strip_slash_remote_dev_prefix}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/${latest_snapshot_identifier}"
aws s3 sync "s3://env-rdev-wmg/${strip_slash_remote_dev_prefix}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/${latest_snapshot_identifier}" "${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH}/${strip_slash_remote_dev_prefix}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/${latest_snapshot_identifier}"
echo aws s3 cp "s3://env-rdev-wmg/${strip_slash_remote_dev_prefix}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/latest_snapshot_identifier" "${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH}/${strip_slash_remote_dev_prefix}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/latest_snapshot_identifier"
aws s3 cp "s3://env-rdev-wmg/${strip_slash_remote_dev_prefix}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/latest_snapshot_identifier" "${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH}/${strip_slash_remote_dev_prefix}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/latest_snapshot_identifier"
elif [[ "${DEPLOYMENT_STAGE}" == "dev" || "${DEPLOYMENT_STAGE}" == "staging" || "${DEPLOYMENT_STAGE}" == "prod" ]]; then
echo "| Downloading WMG data snapshot for deployment env: ${DEPLOYMENT_STAGE} from S3 to filesystem path: ${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH}"
latest_snapshot_identifier=$(aws s3 cp "s3://cellxgene-wmg-${DEPLOYMENT_STAGE}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/latest_snapshot_identifier" -)
echo aws s3 sync "s3://cellxgene-wmg-${DEPLOYMENT_STAGE}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/${latest_snapshot_identifier}" "${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/${latest_snapshot_identifier}"
aws s3 sync "s3://cellxgene-wmg-${DEPLOYMENT_STAGE}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/${latest_snapshot_identifier}" "${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/${latest_snapshot_identifier}"
echo aws s3 cp "s3://cellxgene-wmg-${DEPLOYMENT_STAGE}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/latest_snapshot_identifier" "${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/latest_snapshot_identifier"
aws s3 cp "s3://cellxgene-wmg-${DEPLOYMENT_STAGE}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/latest_snapshot_identifier" "${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH}/snapshots/${LATEST_READER_SNAPSHOT_SCHEMA_VERSION}/latest_snapshot_identifier"
else
echo "| Skipping downloading WMG data snapshot for deployment env: ${DEPLOYMENT_STAGE}..."
fi
echo "| Finished downloading WMG data snapshot from S3 to filesystem path: ${WMG_SNAPSHOT_FS_CACHE_ROOT_PATH} for valid deployment environments"
# If user passed a command line, run it in place of the server
if [ $# -ne 0 ]; then
exec "$@"
fi
if [ "${DEPLOYMENT_STAGE}" == "test" ]; then
# Use locally-generated cert for HTTPS in containerized local environment; deployed envs use ELB
HTTPS_CERT_AND_KEY="--certfile /tmp/pkcs12/server.crt --keyfile /tmp/pkcs12/server.key"
fi
# TODO: Fix problem with invoking gunicorn under ddtrace-run. Also make sure when
# DEPLOYMENT_STAGE=test, gunicorn is not invoked under ddtrace-run.
# See ticket: https://github.com/chanzuckerberg/single-cell-data-portal/issues/5819
export DD_GEVENT_PATCH_ALL=true
# Note: Using just 1 worker for dev/test env. Multiple workers are used in deployment envs, as defined in Terraform code.
# Note: keep-alive timeout should always be greater than the idle timeout of the load balancer (60 seconds)
echo "starting gunicorn server"
exec gunicorn ${HTTPS_CERT_AND_KEY} --worker-class gevent --workers 1 --bind 0.0.0.0:5000 backend.api_server.app:app \
--max-requests 10000 --timeout 540 --keep-alive 61 --log-level info