Release/7.15 - Changes Since rapidpro/v7.2.4

.coveragerc

@@ -7,7 +7,7 @@ ignore_errors = True
 precision = 0
 show_missing = True
 skip_covered = True
-exclude_lines = 
+exclude_lines =
     pragma: no cover
     pragma: needs cover
     pragma: can't cover

.dockerignore

@@ -0,0 +1,3 @@
+env
+env_py3
+node_modules

.github/workflows/ci.yml

@@ -66,22 +66,22 @@ jobs:
         npm install
         sudo npm install -g less
         ln -s ${{ github.workspace }}/temba/settings.py.dev ${{ github.workspace }}/temba/settings.py
-        poetry run ./manage.py migrate
+        poetry run python manage.py migrate
         # fetch and extract rp-indexer
         wget https://github.com/nyaruka/rp-indexer/releases/download/v${{ env.rpindexer-version }}/rp-indexer_${{ env.rpindexer-version }}_linux_amd64.tar.gz
         tar -xvf rp-indexer_${{ env.rpindexer-version }}_linux_amd64.tar.gz rp-indexer
         # fetch, extract and start mailroom
-        wget https://github.com/nyaruka/mailroom/releases/download/v${{ env.mailroom-version }}/mailroom_${{ env.mailroom-version }}_linux_amd64.tar.gz
+        wget https://github.com/rapidpro/mailroom/releases/download/v${{ env.mailroom-version }}/mailroom_${{ env.mailroom-version }}_linux_amd64.tar.gz
         tar -xvf mailroom_${{ env.mailroom-version }}_linux_amd64.tar.gz mailroom
         ./mailroom -db=postgres://temba:temba@localhost:5432/temba?sslmode=disable -redis=redis://localhost:6379/10 -log-level=info > mailroom.log &
 
     - name: Run pre-test checks
       run: |
         # test that everything is well formatted, no missing migrations etc
-        poetry run ./code_check.py --debug
+        poetry run python code_check.py --debug
         # check we can collect and compress all static files
-        poetry run ./manage.py collectstatic --noinput --verbosity=0
-        poetry run ./manage.py compress --extension=".haml" --settings=temba.settings_compress
+        poetry run python manage.py collectstatic --noinput --verbosity=0
+        poetry run python manage.py compress --extension=".haml" --settings=temba.settings_compress
 
     - name: Run tests
       run: |
@@ -98,4 +98,5 @@ jobs:
       if: success()
       uses: codecov/codecov-action@v1
       with:
+        token: ${{ secrets.CODECOV_TOKEN }}
         fail_ci_if_error: true

.github/workflows/owasp.yml

@@ -0,0 +1,112 @@
+name: OWASP ZAP Scanning
+on:
+  push:
+    branches:
+      - 'feature/owasp-scanning'
+      - 'dev/*'
+      - 'release/*'
+  pull_request:
+    branches:
+      - 'dev/*'
+      - 'release/*'
+jobs:
+  zap_scan:
+    name: Scan the WEB Application
+    runs-on: ubuntu-latest
+    env:
+      python_version: '3.9.x'
+      postgis_version: '3.1'
+      pg_version: '12'
+      redis_version: '5'
+      node_version: '10'
+      elastic_version: '7.10.1'
+      rpindexer_version: '6.4.0'
+      mailroom_version: '6.5.36'
+      DJANGO_SETTINGS_MODULE: temba.settings_ci
+      email_url: ${{secrets.MAIL_SERVER_ADDRESS}}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Install PostgreSQL and PostGIS
+      uses: nyaruka/[email protected]
+      with:
+        postgresql version: ${{ env.pg_version }}
+        postgis version: ${{ env.postgis_version }}
+        postgresql db: temba
+        postgresql user: temba
+        postgresql password: temba
+        postgresql arguments: '--jit=off'
+
+    - name: Install Linux packages
+      run: |
+        sudo apt update
+        sudo apt install -y --no-install-recommends libgdal-dev gettext
+
+    - name: Install Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ env.python_version }}
+
+    - name: Install Redis
+      uses: zhulik/[email protected]
+      with:
+        redis version: ${{ env.redis_version }}
+
+    - name: Install ElasticSearch
+      uses: nyaruka/elasticsearch-action@master
+      with:
+        elastic version: ${{ env.elastic_version }}
+
+    - name: Install Node
+      uses: actions/setup-node@v1
+      with:
+        node-version: ${{ env.node_version }}
+
+    - name: Initialize environment
+      run: |
+        python -m pip install -U pip poetry
+        poetry install
+        npm install
+        sudo npm install -g less
+        ln -s ${{ github.workspace }}/temba/settings.py.dev ${{ github.workspace }}/temba/settings.py
+        poetry run python manage.py migrate
+
+    - name: Run RP-Indexer
+      run: |
+        # fetch and extract rp-indexer
+        wget https://github.com/nyaruka/rp-indexer/releases/download/v${{ env.rpindexer_version }}/rp-indexer_${{ env.rpindexer_version }}_linux_amd64.tar.gz
+        tar -xvf rp-indexer_${{ env.rpindexer_version }}_linux_amd64.tar.gz rp-indexer
+
+    - name: Run Mailroom
+      run: |
+        # fetch, extract and start mailroom
+        wget https://github.com/rapidpro/mailroom/releases/download/v${{ env.mailroom_version }}/mailroom_${{ env.mailroom_version }}_linux_amd64.tar.gz
+        tar -xvf mailroom_${{ env.mailroom_version }}_linux_amd64.tar.gz mailroom
+        ./mailroom -db=postgres://temba:temba@localhost:5432/temba?sslmode=disable -redis=redis://localhost:6379/10 -log-level=info > mailroom.log &
+
+    - name: Run RapidPro
+      run: |
+        poetry run python manage.py runserver > rapidpro.log &
+
+    - name: ZAP Scan
+      uses: zaproxy/[email protected]
+      with:
+        target: 'http://127.0.0.1:8000'
+        rules_file_name: '.zap/rules.tsv'
+        cmd_options: '-a'
+        allow_issue_writing: false
+
+    - name: Send mail
+      if: ${{ env.email_url }}
+      uses: dawidd6/action-send-mail@v3
+      with:
+        server_address: ${{secrets.MAIL_SERVER_ADDRESS}}
+        server_port: ${{secrets.MAIL_SERVER_PORT}}
+        username: ${{secrets.MAIL_USERNAME}}
+        password: ${{secrets.MAIL_PASSWORD}}
+        subject: Github OWASP Scanning Action job result
+        from: ${{secrets.MAIL_DEFAULT_FROM}}
+        to:  ${{secrets.MAIL_SEND_OWASP_RESULT_TO}}
+        body: Scanning job of ${{github.repository}} completed successfully!
+        attachments: report_html.html

.gitignore

@@ -20,6 +20,7 @@ config/gen
 config/settings.py
 coverage-report
 celerybeat-schedule.db
+celerybeat-schedule
 db
 node_modules
 parser.out
@@ -125,4 +126,17 @@ ansible/site.retry
 .vscode/
 
 # autoenv
-.direnv/
+.direnv/
+
+# CCL Custom
+env_py3/
+rp-indexer/
+media/cache/
+media/flow_pdf/
+media/duplicates_check/
+temba/settings_test.py
+docker.settings.aptible-prod
+docker.settings.aptible-stage
+docker.settings.prod
+Dockerfile
+.env

.platform/nginx/conf.d/proxy.conf

@@ -0,0 +1,7 @@
+client_max_body_size 200M;
+client_header_timeout   300;
+client_body_timeout     300;
+send_timeout            300;
+proxy_read_timeout      300;
+proxy_connect_timeout   300;
+proxy_send_timeout      300;

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        files: 'temba'
+
+  - repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+      - id: black
+        language_version: python3.9
+        args: ['--line-length=119', '--target-version=py36']
+        files: 'temba'
+
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        args: ['--max-line-length=119', '--ignore=E501,F405,T003,E203,W503']
+        files: 'temba'

.zap/rules.tsv

@@ -0,0 +1,2 @@
+10011	IGNORE	(Cookie Without Secure Flag)
+10015	IGNORE	(Incomplete or No Cache-control and Pragma HTTP Header Set)

Dockerrun.aws.json.default

@@ -0,0 +1,12 @@
+{
+  "AWSEBDockerrunVersion": "1",
+  "Image": {
+    "Name": "DOCKER_IMAGE_NAME",
+    "Update": "true"
+  },
+  "Ports": [
+    {
+      "ContainerPort": "8000"
+    }
+  ]
+}

README.md

@@ -1,7 +1,7 @@
 # RapidPro 
 
-[![Build Status](https://github.com/rapidpro/rapidpro/workflows/CI/badge.svg)](https://github.com/rapidpro/rapidpro/actions?query=workflow%3ACI) 
-[![codecov](https://codecov.io/gh/rapidpro/rapidpro/branch/main/graph/badge.svg)](https://codecov.io/gh/rapidpro/rapidpro)
+[![Build Status](https://github.com/greatnonprofits-nfp/rapidpro/workflows/CI/badge.svg)](https://github.com/rapidpro/rapidpro/actions?query=workflow%3ACI) 
+[![codecov](https://codecov.io/gh/greatnonprofits-nfp/rapidpro/branch/main/graph/badge.svg)](https://codecov.io/gh/greatnonprofits-nfp/rapidpro)
 
 RapidPro is a hosted service for visually building interactive messaging applications.
 To learn more, please visit the project site at http://rapidpro.github.io/rapidpro.
@@ -47,7 +47,12 @@ version of these dependencies you need to run with it.
 ## Updating FlowEditor version
 
 ```
-% npm install @nyaruka/flow-editor@whatever-version --save
+% npm install @greatnonprofits-nfp/flow-editor@whatever-version --save
+```
+
+## Installing pre-commit to lint and format the code
+```bash
+pre-commit install
 ```
 
 ### Get Involved

code_check.py

@@ -44,7 +44,7 @@ def status(line):
 
     if not args.skip_flake:
         status("Running flake8")
-        cmd("flake8")
+        cmd("flake8 temba --ignore=E501,F405,T003,E203,W503,F541,E741")
 
     # if any code changes were made, exit with error
     if cmd("git diff temba locale"):

docker/Dockerfile.app

@@ -0,0 +1,48 @@
+FROM greatnonprofits/ccl-base:v4
+
+RUN wget https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem \
+    -O /usr/local/share/ca-certificates/rds.crt
+RUN update-ca-certificates
+
+RUN wget https://ccl-prod.s3.us-west-1.amazonaws.com/phantomjs-2.1.1-linux-x86_64.tar.bz2
+RUN tar xvjf phantomjs-2.1.1-linux-x86_64.tar.bz2 -C /usr/local/share/
+RUN ln -sf /usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/local/bin
+
+RUN mkdir /rapidpro
+WORKDIR /rapidpro
+
+COPY ./pyproject.toml /rapidpro/pyproject.toml
+COPY ./poetry.lock /rapidpro/poetry.lock
+
+RUN pip3 install --upgrade pip setuptools
+RUN pip3 install -U poetry
+
+RUN poetry export --without-hashes --output pip-freeze.txt
+
+RUN pip3 install -r pip-freeze.txt
+
+COPY . /rapidpro
+COPY docker/docker.settings /rapidpro/temba/settings.py
+
+RUN npm install
+
+RUN python3.9 manage.py collectstatic --noinput
+
+RUN echo "daemon off;" >> /etc/nginx/nginx.conf
+
+RUN rm -f /etc/nginx/sites-enabled/default
+RUN ln -sf /rapidpro/docker/nginx.conf /etc/nginx/sites-enabled/
+
+RUN rm -f /rapidpro/temba/settings.pyc
+
+COPY docker/entrypoint.sh /
+RUN chmod +x /entrypoint.sh
+
+RUN ln -s /usr/bin/python3.9 /usr/bin/python
+RUN rm -rf /tmp/* /var/tmp/*[~]$
+
+EXPOSE 8000
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD ["app"]

docker/Dockerfile.celery

@@ -0,0 +1,41 @@
+FROM greatnonprofits/ccl-base:v4
+
+RUN wget https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem \
+    -O /usr/local/share/ca-certificates/rds.crt
+RUN update-ca-certificates
+
+RUN wget https://ccl-prod.s3.us-west-1.amazonaws.com/phantomjs-2.1.1-linux-x86_64.tar.bz2
+RUN tar xvjf phantomjs-2.1.1-linux-x86_64.tar.bz2 -C /usr/local/share/
+RUN ln -sf /usr/local/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/local/bin
+
+RUN mkdir /rapidpro
+WORKDIR /rapidpro
+
+COPY ./pyproject.toml /rapidpro/pyproject.toml
+COPY ./poetry.lock /rapidpro/poetry.lock
+
+RUN pip3 install --upgrade pip setuptools
+RUN pip3 install -U poetry
+
+RUN poetry export --without-hashes --output pip-freeze.txt
+
+RUN pip3 install -r pip-freeze.txt
+
+COPY . /rapidpro
+COPY docker/docker.settings /rapidpro/temba/settings.py
+
+RUN npm install
+
+RUN rm -f /rapidpro/temba/settings.pyc
+
+COPY docker/entrypoint.sh /
+RUN chmod +x /entrypoint.sh
+
+RUN ln -s /usr/bin/python3.9 /usr/bin/python
+RUN rm -rf /tmp/* /var/tmp/*[~]$
+
+EXPOSE 8000
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD ["celery"]

docker/clear-compressor-cache.py

@@ -0,0 +1,24 @@
+import os
+import redis
+
+from django.conf import settings
+from django.core.cache import cache
+
+settings.configure()
+
+key_prefix = cache.make_key('django_compressor')
+
+redis_host = os.environ.get("REDIS_HOST", "localhost")
+redis_port = int(os.environ.get("REDIS_PORT", 6379))
+redis_password = os.environ.get("REDIS_PW", None)
+redis_db = int(os.environ.get("REDIS_DB", 10))
+
+if redis_password is None:
+    redis = redis.Redis(host=redis_host, port=redis_port, db=redis_db)
+else:
+    redis = redis.Redis(host=redis_host, port=redis_port, db=redis_db, password=redis_password)
+
+keys = redis.keys('%s.*' % (key_prefix,))
+for key in keys:
+    redis.delete(key)
+    print('Cleared Django Compressor key: %s' % (key,))