Ray master

.clang-format

@@ -1,6 +1,5 @@
-BasedOnStyle: Chromium
-ColumnLimit: 80
+BasedOnStyle: Google
+ColumnLimit: 90
 DerivePointerAlignment: false
 IndentCaseLabels: false
 PointerAlignment: Right
-SpaceAfterCStyleCast: true

.gitignore

@@ -4,26 +4,17 @@
 /python/build
 /python/dist
 /python/flatbuffers-1.7.1/
-/src/common/thirdparty/redis
-/src/thirdparty/arrow
 /flatbuffers-1.7.1/
-/src/thirdparty/boost/
-/src/thirdparty/boost_1_65_1/
-/src/thirdparty/boost_1_60_0/
-/src/thirdparty/catapult/
-/src/thirdparty/flatbuffers/
-/src/thirdparty/parquet-cpp
 /thirdparty/pkg/
 
 # Files generated by flatc should be ignored
-/src/common/format/*.py
-/src/common/format/*_generated.h
-/src/plasma/format/
-/src/local_scheduler/format/*_generated.h
 /src/ray/gcs/format/*_generated.h
 /src/ray/object_manager/format/*_generated.h
 /src/ray/raylet/format/*_generated.h
 
+# Modin source files
+/python/ray/modin
+
 # Redis temporary files
 *dump.rdb
 
@@ -54,9 +45,6 @@ python/.eggs
 *.dylib
 *.dll
 
-# Cython-generated files
-*.c
-
 # Incremental linking files
 *.ilk
 

.travis.yml

@@ -53,8 +53,8 @@ matrix:
         - sphinx-build -W -b html -d _build/doctrees source _build/html
         - cd ..
         # Run Python linting, ignore dict vs {} (C408), others are defaults
-        - flake8 --exclude=python/ray/core/src/common/flatbuffers_ep-prefix/,python/ray/core/generated/,src/common/format/,doc/source/conf.py,python/ray/cloudpickle/ --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504
-        - .travis/yapf.sh --all
+        - flake8 --exclude=python/ray/core/generated/,doc/source/conf.py,python/ray/cloudpickle/ --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
+        - .travis/format.sh --all
 
     - os: linux
       dist: trusty
@@ -69,16 +69,9 @@ matrix:
 
       script:
         - cd build
-        - bash ../src/common/test/run_valgrind.sh
-        - bash ../src/plasma/test/run_valgrind.sh
-        - bash ../src/local_scheduler/test/run_valgrind.sh
         - bash ../src/ray/test/run_object_manager_valgrind.sh
         - cd ..
 
-        - python ./python/ray/plasma/test/test.py valgrind
-        - python ./python/ray/local_scheduler/test/test.py valgrind
-        # - python ./python/ray/global_scheduler/test/test.py valgrind
-
     # Build Linux wheels.
     - os: linux
       dist: trusty
@@ -107,63 +100,6 @@ matrix:
       env:
         - PYTHON=3.5
         - RAY_USE_NEW_GCS=on
-        - RAY_USE_XRAY=1
-
-    - os: linux
-      dist: trusty
-      env: PYTHON=3.5 RAY_USE_XRAY=1
-      install:
-        - ./.travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./.travis/install-ray.sh
-        - ./.travis/install-cython-examples.sh
-      script:
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        # The following is needed so cloudpickle can find some of the
-        # class definitions: The main module of tests that are run
-        # with pytest have the same name as the test file -- and this
-        # module is only found if the test directory is in the PYTHONPATH.
-        - export PYTHONPATH="$PYTHONPATH:./test/"
-
-        - python -m pytest -v python/ray/common/test/test.py
-        - python -m pytest -v python/ray/common/redis_module/runtest.py
-        - python -m pytest -v python/ray/plasma/test/test.py
-        # - python -m pytest -v python/ray/local_scheduler/test/test.py
-        # - python -m pytest -v python/ray/global_scheduler/test/test.py
-
-        - python -m pytest -v python/ray/test/test_global_state.py
-        - python -m pytest -v python/ray/test/test_queue.py
-        - python -m pytest -v test/xray_test.py
-
-        - python -m pytest -v test/runtest.py
-        - python -m pytest -v test/array_test.py
-        - python -m pytest -v test/actor_test.py
-        - python -m pytest -v test/autoscaler_test.py
-        - python -m pytest -v test/tensorflow_test.py
-        - python -m pytest -v test/failure_test.py
-        - python -m pytest -v test/microbenchmarks.py
-        - python -m pytest -v test/stress_tests.py
-        - pytest test/component_failures_test.py
-        - python test/multi_node_test.py
-        - python -m pytest -v test/recursion_test.py
-        - pytest test/monitor_test.py
-        - python -m pytest -v test/cython_test.py
-        - python -m pytest -v test/credis_test.py
-
-        # ray tune tests
-        - python python/ray/tune/test/dependency_test.py
-        - python -m pytest -v python/ray/tune/test/trial_runner_test.py
-        - python -m pytest -v python/ray/tune/test/trial_scheduler_test.py
-        - python -m pytest -v python/ray/tune/test/experiment_test.py
-        - python -m pytest -v python/ray/tune/test/tune_server_test.py
-        - python -m pytest -v python/ray/tune/test/ray_trial_executor_test.py
-        - python -m pytest -v python/ray/tune/test/automl_searcher_test.py
-
-        # ray rllib tests
-        - python -m pytest -v python/ray/rllib/test/test_catalog.py
-        - python -m pytest -v python/ray/rllib/test/test_filters.py
-        - python -m pytest -v python/ray/rllib/test/test_optimizers.py
-        - python -m pytest -v python/ray/rllib/test/test_evaluators.py
 
 
 install:
@@ -181,12 +117,10 @@ install:
   - ./src/ray/raylet/lineage_cache_test
   - ./src/ray/raylet/task_dependency_manager_test
   - ./src/ray/raylet/reconstruction_policy_test
+  - ./src/ray/raylet/client_connection_test
   - ./src/ray/util/logging_test --gtest_filter=PrintLogTest*
   - ./src/ray/util/signal_test
 
-  - bash ../src/common/test/run_tests.sh
-  - bash ../src/plasma/test/run_tests.sh
-  - bash ../src/local_scheduler/test/run_tests.sh
   - cd ..
 
 script:
@@ -197,14 +131,27 @@ script:
   # module is only found if the test directory is in the PYTHONPATH.
   - export PYTHONPATH="$PYTHONPATH:./test/"
 
-  - python -m pytest -v python/ray/common/test/test.py
-  - python -m pytest -v python/ray/common/redis_module/runtest.py
-  - python -m pytest -v python/ray/plasma/test/test.py
-  - python -m pytest -v python/ray/local_scheduler/test/test.py
-  - python -m pytest -v python/ray/global_scheduler/test/test.py
+  # ray tune tests
+  - python python/ray/tune/test/dependency_test.py
+  - python -m pytest -v python/ray/tune/test/trial_runner_test.py
+  - python -m pytest -v python/ray/tune/test/trial_scheduler_test.py
+  - python -m pytest -v python/ray/tune/test/experiment_test.py
+  - python -m pytest -v python/ray/tune/test/tune_server_test.py
+  - python -m pytest -v python/ray/tune/test/ray_trial_executor_test.py
+  - python -m pytest -v python/ray/tune/test/automl_searcher_test.py
+
+  # ray rllib tests
+  - python -m pytest -v python/ray/rllib/test/test_catalog.py
+  - python -m pytest -v python/ray/rllib/test/test_filters.py
+  - python -m pytest -v python/ray/rllib/test/test_optimizers.py
+  - python -m pytest -v python/ray/rllib/test/test_evaluators.py
+
+  # Python3.5+ only. Otherwise we will get `SyntaxError` regardless of how we set the tester.
+  - python -c 'import sys;exit(sys.version_info>=(3,5))' || python -m pytest -v python/ray/experimental/test/async_test.py
 
   - python -m pytest -v python/ray/test/test_global_state.py
   - python -m pytest -v python/ray/test/test_queue.py
+  - python -m pytest -v python/ray/test/test_ray_init.py
   - python -m pytest -v test/xray_test.py
 
   - python -m pytest -v test/runtest.py
@@ -216,26 +163,19 @@ script:
   - python -m pytest -v test/microbenchmarks.py
   - python -m pytest -v test/stress_tests.py
   - python -m pytest -v test/component_failures_test.py
-  - python test/multi_node_test.py
+  - python -m pytest -v test/multi_node_test.py
+  - python -m pytest -v test/multi_node_test_2.py
   - python -m pytest -v test/recursion_test.py
   - python -m pytest -v test/monitor_test.py
   - python -m pytest -v test/cython_test.py
   - python -m pytest -v test/credis_test.py
+  - python -m pytest -v test/node_manager_test.py
 
-  # ray tune tests
-  - python python/ray/tune/test/dependency_test.py
-  - python -m pytest -v python/ray/tune/test/trial_runner_test.py
-  - python -m pytest -v python/ray/tune/test/trial_scheduler_test.py
-  - python -m pytest -v python/ray/tune/test/experiment_test.py
-  - python -m pytest -v python/ray/tune/test/tune_server_test.py
-  - python -m pytest -v python/ray/tune/test/ray_trial_executor_test.py
-  - python -m pytest -v python/ray/tune/test/automl_searcher_test.py
+  # ray temp file tests
+  - python -m pytest -v test/tempfile_test.py
 
-  # ray rllib tests
-  - python -m pytest -v python/ray/rllib/test/test_catalog.py
-  - python -m pytest -v python/ray/rllib/test/test_filters.py
-  - python -m pytest -v python/ray/rllib/test/test_optimizers.py
-  - python -m pytest -v python/ray/rllib/test/test_evaluators.py
+  # modin test files
+  - python python/ray/test/test_modin.py
 
 deploy:
   - provider: s3

.travis/yapf.sh → .travis/format.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# YAPF + Clang formatter (if installed). This script formats all changed files from the last mergebase.
+# You are encouraged to run this locally before pushing changes for review.
 
 # Cause the script to exit if a single command fails
 set -eo pipefail
@@ -28,7 +30,6 @@ YAPF_EXCLUDES=(
     '--exclude' 'python/build/*'
     '--exclude' 'python/ray/pyarrow_files/*'
     '--exclude' 'python/ray/core/src/ray/gcs/*'
-    '--exclude' 'python/ray/common/thirdparty/*'
 )
 
 # Format specified files
@@ -50,6 +51,18 @@ format_changed() {
     if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.py' &>/dev/null; then
         git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' | xargs -P 5 \
              yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
+        if which flake8 >/dev/null; then
+            git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.py' | xargs -P 5 \
+                 flake8 --exclude=python/ray/core/generated/,doc/source/conf.py,python/ray/cloudpickle/ \
+                    --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
+        fi
+    fi
+
+    if which clang-format >/dev/null; then
+        if ! git diff --diff-filter=ACM --quiet --exit-code "$MERGEBASE" -- '*.cc' '*.h' &>/dev/null; then
+            git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.cc' '*.h' | xargs -P 5 \
+                 clang-format -i
+        fi
     fi
 }
 

.travis/install-dependencies.sh

@@ -24,17 +24,17 @@ if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \
-    feather-format lxml openpyxl xlrd
+  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+    feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
   sudo apt-get update
   sudo apt-get install -y cmake pkg-config python-dev python-numpy build-essential autoconf curl libtool unzip
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \
-    feather-format lxml openpyxl xlrd
+  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+    feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout
 elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
   which -s brew
@@ -50,8 +50,8 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \
-    feather-format lxml openpyxl xlrd
+  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+    feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
   which -s brew
@@ -67,8 +67,8 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.22 requests \
-    feather-format lxml openpyxl xlrd
+  pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+    feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout
 elif [[ "$LINT" == "1" ]]; then
   sudo apt-get update
   sudo apt-get install -y cmake build-essential autoconf curl libtool unzip

.travis/test-wheels.sh

@@ -56,23 +56,25 @@ if [[ "$platform" == "linux" ]]; then
 
   # Check that the other wheels are present.
   NUMBER_OF_WHEELS=$(ls -1q $ROOT_DIR/../.whl/*.whl | wc -l)
-  if [[ "$NUMBER_OF_WHEELS" != "4" ]]; then
+  if [[ "$NUMBER_OF_WHEELS" != "5" ]]; then
     echo "Wrong number of wheels found."
     ls -l $ROOT_DIR/../.whl/
-    exit 1
+    exit 2
   fi
 
 elif [[ "$platform" == "macosx" ]]; then
   MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
   PY_MMS=("2.7"
           "3.4"
           "3.5"
-          "3.6")
+          "3.6"
+          "3.7")
   # This array is just used to find the right wheel.
   PY_WHEEL_VERSIONS=("27"
                      "34"
                      "35"
-                     "36")
+                     "36"
+                     "37")
 
   for ((i=0; i<${#PY_MMS[@]}; ++i)); do
     PY_MM=${PY_MMS[i]}
@@ -92,5 +94,5 @@ elif [[ "$platform" == "macosx" ]]; then
   done
 else
   echo "Unrecognized environment."
-  exit 1
+  exit 3
 fi

CMakeLists.txt

@@ -82,18 +82,15 @@ include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
 include_directories("${CMAKE_CURRENT_LIST_DIR}/src/")
 
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/ray/)
-add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/common/)
-add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/)
-add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/local_scheduler/)
-add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/global_scheduler/)
 
 # final target copy_ray
 add_custom_target(copy_ray ALL)
 
 # copy plasma_store_server
 add_custom_command(TARGET copy_ray POST_BUILD
+  COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/src/plasma
   COMMAND ${CMAKE_COMMAND} -E
-    copy ${ARROW_HOME}/bin/plasma_store_server ${CMAKE_CURRENT_BINARY_DIR}/src/plasma)
+    copy ${ARROW_HOME}/bin/plasma_store_server ${CMAKE_CURRENT_BINARY_DIR}/src/plasma/)
 
 if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   # add pyarrow as the dependency
@@ -102,12 +99,9 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   # NOTE: The lists below must be kept in sync with ray/python/setup.py.
 
   set(ray_file_list
-        "src/common/thirdparty/redis/src/redis-server"
-        "src/common/redis_module/libray_redis_module.so"
-        "src/plasma/plasma_manager"
-        "src/local_scheduler/local_scheduler"
-        "src/local_scheduler/liblocal_scheduler_library_python.so"
-        "src/global_scheduler/global_scheduler"
+        "src/ray/thirdparty/redis/src/redis-server"
+        "src/ray/gcs/redis_module/libray_redis_module.so"
+        "src/ray/raylet/liblocal_scheduler_library_python.so"
         "src/ray/raylet/raylet_monitor"
         "src/ray/raylet/raylet")
 
@@ -117,7 +111,10 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
     list(APPEND ray_file_list "src/credis/redis/src/redis-server")
   endif()
 
-  if (DEFINED ENV{INCLUDE_UI} AND "$ENV{INCLUDE_UI}" STREQUAL "1")
+  # The goal of the if statement below is to require the catapult files to be
+  # present INCLUDE_UI=1 is set and to include the UI files if they are present.
+  # This should match the logic in build_ui.sh.
+  if (EXISTS "${CMAKE_BINARY_DIR}/src/catapult_files/index.html" OR "$ENV{INCLUDE_UI}" STREQUAL "1")
     list(APPEND ray_file_list "src/catapult_files/index.html")
     list(APPEND ray_file_list "src/catapult_files/trace_viewer_full.html")
   endif()
@@ -154,5 +151,6 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
 
   # copy libplasma_java files
   add_custom_command(TARGET copy_ray POST_BUILD
-    COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma")
+    COMMAND bash -c "mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/src/plasma"
+    COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma/")
 endif()