Skip to content

Commit

Permalink
Organized slurm output for iss #80
Browse files Browse the repository at this point in the history
  • Loading branch information
@ebrintn
ebrintn committed Sep 30, 2024
1 parent ab5a23b commit 485a5e1
Show file tree
Hide file tree
Showing 6 changed files with 48 additions and 18 deletions.
Binary file modified tests/__pycache__/testReadInput.cpython-310.pyc
View file
Binary file not shown.
38 changes: 28 additions & 10 deletions tests/testReadInput.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,8 +209,8 @@ def test_process_filenames(self):
tree_name = "tests/testFiles/test_tree"

#Test process with new directories, no backup, ensure folders created
processed_output = self.input_reader.process_filenames(tree_name + ".txt", False)
correct_output = (self.test_file_path + "/slimScripts/test_tree", self.test_file_path + "/test_tree", None)
processed_output = self.input_reader.process_filenames(tree_name + ".txt", False, False)
correct_output = [self.test_file_path + "/slimScripts/test_tree", self.test_file_path + "/test_tree", None, None]
self.assertTrue(processed_output == correct_output)
self.assertTrue(os.path.exists(self.test_file_path + "/slimScripts/"))
self.assertTrue(os.path.exists(self.test_file_path + "/aa_FASTA/"))
Expand All @@ -219,9 +219,9 @@ def test_process_filenames(self):
#Test process with existing directories, no backup
with mock.patch.object(builtins, 'input', lambda _: 'y'):
with redirect_stdout(io.StringIO()) as sout:
processed_output2 = self.input_reader.process_filenames(tree_name + ".txt", False)
processed_output2 = self.input_reader.process_filenames(tree_name + ".txt", False, False)

sout_correct = sout.getvalue() == "using the same nuc_FASTA folder\nusing the same aa_FASTA folder\n"
sout_correct = sout.getvalue() == "using the same nuc_FASTA folder\nusing the same aa_FASTA folder\n"
sout.close()
correct = processed_output2 == correct_output and sout_correct
assert correct
Expand All @@ -233,31 +233,49 @@ def test_process_filenames(self):
os.rmdir(self.test_file_path + "/nuc_FASTA/")

#Test process with new directories, include backup, ensure folders created
processed_output3 = self.input_reader.process_filenames(tree_name + ".txt", True)
correct_output2 = (self.test_file_path + "/slimScripts/test_tree", self.test_file_path + "/test_tree", self.test_file_path + "/backupFiles")
processed_output3 = self.input_reader.process_filenames(tree_name + ".txt", True, False)
correct_output2 = [self.test_file_path + "/slimScripts/test_tree", self.test_file_path + "/test_tree", self.test_file_path + "/backupFiles", None]
self.assertTrue(processed_output3 == correct_output2)
self.assertTrue(os.path.exists(self.test_file_path + "/slimScripts/"))
self.assertTrue(os.path.exists(self.test_file_path + "/aa_FASTA/"))
self.assertTrue(os.path.exists(self.test_file_path + "/nuc_FASTA/"))
self.assertTrue(os.path.exists(self.test_file_path + "/backupFiles/"))


# Remove directories to test with a backup
os.rmdir(self.test_file_path + "/slimScripts/")
os.rmdir(self.test_file_path + "/aa_FASTA/")
os.rmdir(self.test_file_path + "/nuc_FASTA/")
# os.rmdir(self.test_file_path + "/backupFiles/")

# Test process with new directories, include hpc, ensure folders created
processed_output5 = self.input_reader.process_filenames(tree_name + ".txt", False, True)
correct_output6 = [self.test_file_path + "/slimScripts/test_tree", self.test_file_path + "/test_tree", None, self.test_file_path + "/slurmOutput"]
self.assertTrue(processed_output3 == correct_output2)
self.assertTrue(os.path.exists(self.test_file_path + "/slimScripts/"))
self.assertTrue(os.path.exists(self.test_file_path + "/aa_FASTA/"))
self.assertTrue(os.path.exists(self.test_file_path + "/nuc_FASTA/"))
self.assertTrue(os.path.exists(self.test_file_path + "/slurmOutput/"))


# Test process with existing directories, no backup
with mock.patch.object(builtins, 'input', lambda _: 'y'):
with redirect_stdout(io.StringIO()) as sout:
processed_output4 = self.input_reader.process_filenames(tree_name + ".txt", True)
processed_output4 = self.input_reader.process_filenames(tree_name + ".txt", True, True)

sout_correct = sout.getvalue() == "using the same nuc_FASTA folder\nusing the same aa_FASTA folder\nusing same backup folder\n"
sout_correct = sout.getvalue() == "using the same nuc_FASTA folder\nusing the same aa_FASTA folder\nusing same backup folder\nusing same slurm folder\n"
sout_val = sout.getvalue()
sout.close()
correct = processed_output4 == correct_output2 and sout_correct
correct = processed_output4 == [self.test_file_path + "/slimScripts/test_tree",
self.test_file_path + "/test_tree", self.test_file_path + "/backupFiles", self.test_file_path + "/slurmOutput"] and sout_correct
assert correct

# Remove directories
os.rmdir(self.test_file_path + "/slimScripts")
os.rmdir(self.test_file_path + "/aa_FASTA/")
os.rmdir(self.test_file_path + "/nuc_FASTA/")
os.rmdir(self.test_file_path + "/backupFiles/")
os.rmdir(self.test_file_path + "/slurmOutput/")

def test_make_param_dict(self):

Expand Down Expand Up @@ -300,7 +318,7 @@ def test_make_param_dict(self):
def test_save_input(self):
#Make parameter dict and file outputs - already tested
param_dict = self.input_reader.make_param_dict(self.arguments)
param_dict["filenames"] = self.input_reader.process_filenames("tests/testFiles/test_tree.txt", False)
param_dict["filenames"] = self.input_reader.process_filenames("tests/testFiles/test_tree.txt", False, False)

#Test default
self.input_reader.save_input(param_dict)
Expand Down
Binary file modified utils/__pycache__/readInput.cpython-310.pyc
View file
Binary file not shown.
Binary file modified utils/__pycache__/writeSLiMHPC.cpython-310.pyc
View file
Binary file not shown.
23 changes: 17 additions & 6 deletions utils/readInput.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def process_input(self):
sys.exit(0)

self.param_dict = self.make_param_dict(args)
self.param_dict["filenames"] = self.process_filenames(args.input_tree, args.backup)
self.param_dict["filenames"] = self.process_filenames(args.input_tree, args.backup, args.high_performance_computing)



Expand Down Expand Up @@ -169,7 +169,7 @@ def check_arguments (self, arguments):


#Read input tree to make output file names
def process_filenames(self, tree_name, backup):
def process_filenames(self, tree_name, backup, hpc):

# Find where data needs to be output to, set up documents and folders accordingly accordingly
output_file_start = os.getcwd() + '/' + tree_name.split('.')[0]
Expand Down Expand Up @@ -202,12 +202,13 @@ def process_filenames(self, tree_name, backup):
try:
os.mkdir(aa_FASTA_files_directory)
except OSError:
print("using the same aa_FASTA folder")
print("using the same aa_FASTA folder")


#Set up where files will be output to
output_filename = output_files_directory + "/" + split_starting_output[-1]

filenames = [output_filename, output_file_start, None, None]

#Make backup folder
if(backup):
Expand All @@ -217,9 +218,19 @@ def process_filenames(self, tree_name, backup):
except OSError:
print("using same backup folder")

return((output_filename, output_file_start, backup_files_directory))
else:
return((output_filename, output_file_start, None))
filenames [2] = backup_files_directory

#Make folder for output and err files
if(hpc):
slurm_directiory = "/".join(split_starting_output[0:(len(split_starting_output)-1)]) + "/slurmOutput"
try:
os.mkdir(slurm_directiory)
except OSError:
print("using same slurm folder")
filenames [3] = slurm_directiory


return filenames



Expand Down
5 changes: 3 additions & 2 deletions utils/writeSLiMHPC.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,9 @@ def create_scripts(self, population_parameters):
#Write out batch file to run slim script
batch_file = open(self.start_params["filenames"][0] + "_" + pop_name + ".sh", "w")
batch_file.write("#!/bin/sh\n\n#SBATCH -J SLiM_Simulation_" + pop_name + "\n#SBATCH -t " + population_parameters["time"] +
"\n#SBATCH -p " + population_parameters["partition"] + "\n#SBATCH -o " + pop_name + ".out\n#SBATCH -e " + pop_name +
".err\n#SBATCH -n 1" + "\n\nslim \'" + self.start_params["filenames"][0] + "_" + pop_name +".slim\'")
"\n#SBATCH -p " + population_parameters["partition"] + "\n#SBATCH -o " + self.start_params["filenames"][3] + "/" + pop_name + ".out" +
"\n#SBATCH -e " + self.start_params["filenames"][3] + "/" + pop_name + ".err" +
"\n#SBATCH -n 1" + "\n\nslim \'" + self.start_params["filenames"][0] + "_" + pop_name +".slim\'")
batch_file.close()

#Open slim script
Expand Down

0 comments on commit 485a5e1

Please sign in to comment.