[WIP] Bumping deps to an intermediary step compatible with Python3.11 (…

…#4005)

### Motivation
Some of our current dependencies are incompatible with Python 3.11. This
PR attempts to bump the least possible amount of dependencies (more
specifically, grpcio, grpcio-tools, psutil and markupsafe), so we can
isolate deployment issues between dependency bumps and the python
upgrade itself.

Pipfile

@@ -8,11 +8,11 @@ cryptography = "==3.4.8"
 crcmod = "==1.7"
 future = "==0.17.1"
 protobuf = "==3.20.3"
-psutil = "==5.7.0"
+psutil = "==5.9.4"
 
 [dev-packages]
 Fabric = "==1.14.1"
-grpcio-tools = "==1.38.0"
+grpcio-tools = "==1.49.0"
 gunicorn = "*"
 isort = "*"
 nodeenv = "==1.0.0"

src/Pipfile

@@ -5,7 +5,11 @@ name = "pypi"
 
 [packages]
 antlr4-python3-runtime = "==4.7.2"
+bcrypt = "==4.1.2"
+certifi = "==2024.2.2"
+cffi = "==1.15.1"
 cryptography = "==37.0.4"
+future = "==0.17.1"
 google-api-python-client = "==2.24.0"
 google-auth = "==1.25.0"
 google-auth-oauthlib = "==0.4.1"
@@ -15,33 +19,36 @@ google-cloud-datastore = "==1.12.0"
 google-cloud-logging = "==1.15.0"
 google-cloud-monitoring = "==0.36.0"
 google-cloud-ndb = "==1.4.0"
-google-cloud-profiler = "4.1.0"
+google-cloud-profiler = "==4.1.0"
 google-cloud-secret-manager = "==2.17.0"
 google-cloud-storage = "==1.37.0"
-grpcio = "==1.38.0"
+google-crc32c = "==1.5.0"
+grpcio = "==1.49.0"
 httplib2 = "==0.19.0"
 jira = "==2.0.0"
 mozprocess = "==1.3.1"
 oauth2client = "==4.1.3"
+psutil = "==5.9.4"
 protobuf = "==3.20.3"
 pygithub = "==1.55"
 pyOpenSSL = "==22.0.0"
 python-dateutil = "==2.8.1"
-pytz = "==2023.3"
 PyYAML = "==6.0"
+pytz = "==2023.3"
 redis = "==3.3.11"
 requests = "==2.21.0"
 sendgrid = "==6.0.4"
+wrapt = "==1.16.0"
+
 # Hack: We are using this to specify App Engine packages,
 # since there is no support for custom package sets.
-future = "==0.17.1"
-
 [dev-packages]
+charset-normalizer = "==3.3.2"
 firebase-admin = "==2.16.0"
-Flask = "==1.1.2"
+Flask = "==2.2.2"
+google-cloud-firestore = "==1.9.2"
 itsdangerous = "==2.0.1"
-Jinja2 = "==2.11.3"
-markupsafe = "==2.0.1"
+Jinja2 = "==3.1.4"
 PyJWT = "==2.7.0"
 requests-toolbelt = "==0.9.1"
-werkzeug = "==2.0.3"
+werkzeug = "==2.2.2"

src/appengine/handlers/base_handler.py

@@ -99,7 +99,7 @@ def encode_json(value):
 _JINJA_ENVIRONMENT = jinja2.Environment(
     loader=jinja2.FileSystemLoader(
         os.path.join(os.path.dirname(__file__), '..', 'templates')),
-    extensions=['jinja2.ext.autoescape'],
+    extensions=[],
     autoescape=True)
 _MENU_ITEMS = []
 

src/appengine/handlers/testcase_detail/show.py

@@ -17,8 +17,9 @@
 import html
 import re
 
-from flask import request
+import flask
 import jinja2
+import markupsafe
 
 from clusterfuzz._internal.base import utils
 from clusterfuzz._internal.build_management import revisions
@@ -297,7 +298,7 @@ def convert_to_lines(raw_stacktrace, crash_state_lines, crash_type):
   raw_lines = raw_stacktrace.splitlines()
 
   frames = get_stack_frames(crash_state_lines)
-  escaped_frames = [jinja2.escape(f) for f in frames]
+  escaped_frames = [markupsafe.escape(f) for f in frames]
   combined_frames = frames + escaped_frames
 
   # Certain crash types have their own customized frames that are not related to
@@ -628,7 +629,7 @@ class DeprecatedHandler(base_handler.Handler):
 
   def get(self):
     """Serve the redirect to the current test case detail page."""
-    testcase_id = request.args.get('key')
+    testcase_id = flask.request.args.get('key')
     if not testcase_id:
       raise helpers.EarlyExitError('No testcase key provided.', 400)
 
@@ -642,5 +643,5 @@ class RefreshHandler(base_handler.Handler):
   @handler.oauth
   def post(self):
     """Serve the testcase detail JSON."""
-    testcase_id = request.get('testcaseId')
+    testcase_id = flask.request.get('testcaseId')
     return self.render_json(get_testcase_detail_by_id(testcase_id))

src/clusterfuzz/_internal/tests/appengine/handlers/base_handler_test.py

@@ -146,7 +146,7 @@ def test_forbidden_not_logged_in(self):
     app = webtest.TestApp(flaskapp)
     response = app.get('/', expect_errors=True)
     self.assertEqual(response.status_int, 302)
-    self.assertEqual('http://localhost/login?dest=http%3A%2F%2Flocalhost%2F',
+    self.assertEqual('/login?dest=http%3A%2F%2Flocalhost%2F',
                      response.headers['Location'])
 
   def test_forbidden_logged_in(self):
@@ -165,8 +165,8 @@ def test_redirect_another_page(self):
     flaskapp.add_url_rule('/', view_func=FlaskRedirectHandler.as_view('/'))
     app = webtest.TestApp(flaskapp)
     response = app.get('/?redirect=%2Fanother-page')
-    self.assertEqual('http://localhost/another-page',
-                     response.headers['Location'])
+    self.assertEqual(response.status_int, 302)
+    self.assertEqual('/another-page', response.headers['Location'])
 
   def test_redirect_another_domain(self):
     """Test redirect to another domain."""

src/local/butler/constants.py

@@ -37,9 +37,8 @@
 # Supported Platforms and ABIS (newer to older order).
 PLATFORMS = collections.OrderedDict([
     ('windows', 'win_amd64'),
-    ('macos', ('macosx_10_14_x86_64', 'macosx_10_9_x86_64',
-               'macosx_10_12_x86_64')),
-    ('linux', ('manylinux2010_x86_64', 'manylinux1_x86_64')),
+    ('macos', ('macosx_10_14_x86_64', 'macosx_10_12_x86_64')),
+    ('linux', ('manylinux2014_x86_64')),
 ])
 
 if sys.version_info.major == 3 and sys.version_info.minor == 7:

src/platform_requirements.txt

@@ -1,2 +1,3 @@
-grpcio==1.38.0
+grpcio==1.49.0
 protobuf==3.20.3
+psutil==5.9.4