From 8ed8c48adc1e359fd0edc94ac0265329160a97e0 Mon Sep 17 00:00:00 2001 From: vighnesh_wednesday Date: Mon, 18 Dec 2023 13:26:46 +0530 Subject: [PATCH] changed formatting --- app/environment.py | 4 +++- tests/test_environment.py | 8 ++++---- tests/test_environment_failure.py | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/app/environment.py b/app/environment.py index a93c6b8..690d42f 100644 --- a/app/environment.py +++ b/app/environment.py @@ -36,7 +36,9 @@ def get_dataframes(databricks: bool, spark, directory_path: str): if databricks: csv_files = [ - file for file in os.listdir(directory_path) if file.endswith(".csv") + file + for file in os.listdir("/dbfs" + directory_path) + if file.endswith(".csv") ] else: cmd = f"aws s3 ls {directory_path}" diff --git a/tests/test_environment.py b/tests/test_environment.py index 8ccfcba..7bf92ae 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -11,7 +11,7 @@ class TestSetKeysGetSpark(unittest.TestCase): - @patch("app.environment.cg.init_glue") + @patch("app.connect_glue.init_glue") @patch("app.environment.cd.create_mount") @patch("app.environment.dotenv.load_dotenv") def test_databricks_environment( @@ -43,7 +43,7 @@ def test_databricks_environment( mock_init_glue.assert_not_called() mock_load_dotenv.assert_not_called() - @patch("app.environment.cg.init_glue") + @patch("app.connect_glue.init_glue") @patch("app.environment.cd.create_mount") @patch("app.environment.dotenv.load_dotenv") def test_glue_local_environment( @@ -66,7 +66,7 @@ def test_glue_local_environment( mock_init_glue.assert_called_once() mock_load_dotenv.assert_called_once() - @patch("app.environment.cg.init_glue") + @patch("app.connect_glue.init_glue") @patch("app.environment.cd.create_mount") @patch("app.environment.dotenv.load_dotenv") def test_glue_online_environment( @@ -116,7 +116,7 @@ def test_databricks_dataframes(self, mock_run, mock_listdir, mock_create_frame): # Assertions self.assertEqual(len(result_df_list), 2) - mock_listdir.assert_called_with(directory_path) + mock_listdir.assert_called_with("/dbfs" + directory_path) mock_create_frame.assert_any_call(spark, "/mnt/rawdata/file1.csv") mock_create_frame.assert_any_call(spark, "/mnt/rawdata/file2.csv") mock_run.assert_not_called() diff --git a/tests/test_environment_failure.py b/tests/test_environment_failure.py index 7967633..f830b04 100644 --- a/tests/test_environment_failure.py +++ b/tests/test_environment_failure.py @@ -11,7 +11,7 @@ class TestSetKeysGetSpark(unittest.TestCase): - @patch("app.environment.cg.init_glue") + @patch("app.connect_glue.init_glue") @patch("app.environment.cd.create_mount") @patch("app.environment.dotenv.load_dotenv") def test_databricks_environment_failure( @@ -35,7 +35,7 @@ def test_databricks_environment_failure( mock_init_glue.assert_not_called() mock_load_dotenv.assert_not_called() - @patch("app.environment.cg.init_glue") + @patch("app.connect_glue.init_glue") @patch("app.environment.cd.create_mount") @patch("app.environment.dotenv.load_dotenv") def test_glue_local_environment_failure( @@ -77,7 +77,7 @@ def test_databricks_dataframes_failure(self, mock_create_frame, mock_listdir): # Assert self.assertEqual(str(context.exception), "Error listing directory") - mock_listdir.assert_called_with(directory_path) + mock_listdir.assert_called_with("/dbfs" + directory_path) mock_create_frame.assert_not_called() @patch("app.environment.sw.create_frame")