diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index acc02a36b..e4aacc6dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,9 @@ repos: rev: '' hooks: - id: end-of-file-fixer + exclude: 'setup.cfg' - id: trailing-whitespace + exclude: 'setup.cfg' - id: debug-statements - repo: local hooks: diff --git a/.sync-zenodo-metadata.py b/.sync-zenodo-metadata.py index 8cd45451b..39d869d6c 100755 --- a/.sync-zenodo-metadata.py +++ b/.sync-zenodo-metadata.py @@ -13,6 +13,7 @@ [2] https://citation-file-format.github.io/1.0.3/specifications/#/person-objects """ from dataclasses import dataclass +from typing import Optional import click import json @@ -24,7 +25,7 @@ class Contributor: last_names: str first_names: str affiliation: str - orcid: str = None + orcid: Optional[str] = None @classmethod def from_citation_author(cls, **citation): @@ -41,11 +42,6 @@ def as_zenodo_creator(self): ret['orcid'] = self.orcid.lstrip('https://orcid.org/') return ret - def as_zenodo_contributor(self): - ret = self.as_zenodo_creator() - ret['type'] = 'Other' - return ret - @click.command() @click.pass_context @@ -70,7 +66,7 @@ def sync(ctx, in_place=False, check=True): zenodo = json.loads(file.read()) zenodo_updated = zenodo.copy() zenodo_updated['creators'] = [a.as_zenodo_creator() for a in authors] - zenodo_updated['contributors'] = [c.as_zenodo_contributor() + zenodo_updated['contributors'] = [c.as_zenodo_creator() for c in contributors if c not in authors] for key in ('version', 'keywords'): zenodo_updated[key] = citation[key] diff --git a/.zenodo.json b/.zenodo.json index 5c3599075..1e3b045da 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -3,110 +3,96 @@ { "affiliation": "University of Michigan", "name": "Benjamin Swerdlow", - "orcid": "0000-0001-6240-1430", - "type": "Other" + "orcid": "0000-0001-6240-1430" }, { "affiliation": "University of Michigan", - "name": "Joshua A. Anderson", - "type": "Other" + "name": "Joshua A. Anderson" }, { "affiliation": "University of Michigan", "name": "Jens Glaser", - "orcid": "0000-0003-1852-3849", - "type": "Other" + "orcid": "0000-0003-1852-3849" }, { "affiliation": "University of Michigan", "name": "Timothy C. Moore", - "orcid": "0000-0002-5709-7259", - "type": "Other" + "orcid": "0000-0002-5709-7259" }, { "affiliation": "University of Michigan", "name": "Rose Cersonsky", - "orcid": "0000-0003-4515-3441", - "type": "Other" + "orcid": "0000-0003-4515-3441" }, { "affiliation": "University of Michigan", "name": "Yuan Zhou", - "orcid": "0000-0002-6727-0836", - "type": "Other" + "orcid": "0000-0002-6727-0836" }, { "affiliation": "Vanderbilt University", "name": "Matthew W. Thompson", - "orcid": "0000-0002-1460-3983", - "type": "Other" + "orcid": "0000-0002-1460-3983" }, { "affiliation": "University of Michigan", "name": "Shannon Moran", - "orcid": "0000-0002-3579-3149", - "type": "Other" + "orcid": "0000-0002-3579-3149" }, { "affiliation": "University of Michigan", - "name": "Yannah Melle", - "type": "Other" + "name": "Yannah Melle" }, { "affiliation": "University of Michigan", "name": "Pengji Zhou", - "orcid": "0000-0002-1409-9633", - "type": "Other" + "orcid": "0000-0002-1409-9633" }, { "affiliation": "Vanderbilt University", "name": "Justin Gilmer", - "orcid": "0000-0002-6915-5591", - "type": "Other" + "orcid": "0000-0002-6915-5591" }, { "affiliation": "University of Michigan", "name": "Alyssa Travitz", - "orcid": "0000-0001-5953-8807", - "type": "Other" + "orcid": "0000-0001-5953-8807" }, { "affiliation": "University of Michigan", "name": "Corwin Kerr", - "orcid": "0000-0003-0776-2596", - "type": "Other" + "orcid": "0000-0003-0776-2596" }, { "affiliation": "University of Michigan", "name": "Tobias Dwyer", - "orcid": "0000-0001-6443-7744", - "type": "Other" + "orcid": "0000-0001-6443-7744" }, { "affiliation": "National Institute of Technology, Hamirpur", - "name": "Vishav Sharma", - "type": "Other" + "name": "Vishav Sharma" }, { "affiliation": "Indian Institute of Technology Roorkee", - "name": "Hardik Ojha", - "type": "Other" + "name": "Hardik Ojha" }, { "affiliation": "Birla Institute of Technology and Science, Pilani - Goa Campus", - "name": "Yash Bhardwaj", - "type": "Other" + "name": "Yash Bhardwaj" }, { "affiliation": "", "name": "Jacob Dice", - "orcid": "0000-0002-9479-1661", - "type": "Other" + "orcid": "0000-0002-9479-1661" }, { "affiliation": "Indian Institute of Technology, Gandhinagar", - "name": "Abhavya Chandra", - "type": "Other" + "name": "Abhavya Chandra" + }, + { + "affiliation": "University of Minnesota", + "name": "Ramanish Singh", + "orcid": "0000-0002-7908-060X" } ], "creators": [ @@ -161,5 +147,5 @@ }, "title": "signac-flow", "upload_type": "software", - "version": "0.10.0" + "version": "0.11.0" } diff --git a/CITATION.cff b/CITATION.cff index f4cd969e2..f905e6612 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ --- cff-version: "1.0.3" title: signac-flow -version: 0.10.0 +version: 0.11.0 abstract: | The signac-flow tool provides the basic components to set up simple to complex workflows for projects as part of the signac framework. That includes the definition of data pipelines, execution of data space operations and the submission of operations to high-performance super computers. authors: diff --git a/changelog.txt b/changelog.txt index 9cf32d681..a4f79a9f3 100644 --- a/changelog.txt +++ b/changelog.txt @@ -5,14 +5,15 @@ Changelog The **signac-flow** package follows `semantic versioning `_. The numbers in brackets denote the related GitHub issue and/or pull request. -next -==== +Version 0.11 +============ -next ----- +[0.11.0] -- 2020-10-09 +---------------------- Added +++++ + - Added classes ``_Directives`` and ``_Directive`` that serve as a smart mapping for directives specified by the environment or user (#265, #283). - Added support for `pre-commit hooks `__ (#333). - Add environment profile for University of Minnesota, Minnesota Supercomputing Institute, Mangi supercomputer (#353). @@ -26,6 +27,7 @@ Changed Fixed +++++ + - Serial execution on Summit correctly counts total node requirements (#342). - Fixed performance regression in job submission in large workspaces (#354). @@ -38,6 +40,14 @@ Removed Version 0.10 ============ +[0.10.1] -- 2020-08-20 +---------------------- + +Fixed ++++++ + +- Fix issue with the submission of bundled operations on cluster environments that do not allow slashes ('/') in cluster scheduler job names (#343). + [0.10.0] -- 2020-06-27 ---------------------- @@ -69,6 +79,7 @@ Changed Fixed +++++ + - Ensure that label names are used when displaying status (#263). - Fix node counting for large resource sets on Summit (#294). @@ -128,7 +139,7 @@ Added - Enable option to always show the full traceback with ``show_traceback = on`` within the ``[flow]`` section of the signac configuration (#61, #144). - Add full launcher support for job submission on XSEDE Stampede2 for large parallel single processor jobs (#85, #91). -Fixes +Fixed +++++ - Both the ``nranks`` and ``omp_num_threads`` directives properly support callables (#118). @@ -170,12 +181,12 @@ Added - Add function to automatically print all varying state point parameters in the detailed status view triggered by providing option `-p/--parameters` without arguments (#19, #87). - Add clear environment notification when submitting job scripts (#43, #88). -Fixes +Fixed +++++ - Fix issue where the scheduler status of job-operations would not be properly updated for *ineligible* operations (#96). -Fixes (compute environments) +Fixed (compute environments) ++++++++++++++++++++++++++++ - Fix issue with the TORQUE scheduler that occured when there was no job scheduled at all on the system (for any user) (#92, #93). @@ -195,7 +206,7 @@ Added - Add legend explaining the scheduler-related symbols to the detailed status view (#68). - Allow the specification of the number of tasks per resource set and additional jsrun arguments for Summit scripts. -Fixes (general) +Fixed (general) +++++++++++++++ - Fixes issue where callable cmd-directives were not evaluated (#47). @@ -205,7 +216,7 @@ Fixes (general) - Fixes issue where using the buffered mode would lead to confusing error messages when condition-functions would raise an AttributeError exception. - Fixes issue with erroneous unused-directive-keys-warning. -Fixes (compute environments) +Fixed (compute environments) ++++++++++++++++++++++++++++ - Fixes issues with the Summit environment resource set calculation for parallel operations under specific conditions (#63). diff --git a/doc/conf.py b/doc/conf.py index 355940249..76058037a 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -73,9 +73,9 @@ def __getattr__(cls, name): # built documents. # # The short X.Y version. -version = '0.10.0' +version = '0.11.0' # The full version, including alpha/beta/rc tags. -release = '0.10.0' +release = '0.11.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/flow/environment.py b/flow/environment.py index 604624b91..7f1c754a6 100644 --- a/flow/environment.py +++ b/flow/environment.py @@ -16,7 +16,6 @@ import importlib from collections import OrderedDict import importlib.machinery -import warnings from signac.common import config @@ -227,10 +226,6 @@ def _get_mpi_prefix(cls, operation, parallel): """ if operation.directives.get('nranks'): return '{} -n {} '.format(cls.mpi_cmd, operation.directives['nranks']) - elif operation.directives.get('ngpu', 0) > 1: - warnings.warn("Setting ngpu directive without nranks will no longer use MPI " - "in version 0.11.", DeprecationWarning) - return '{} -n {}'.format(cls.mpi_cmd, operation.directives['ngpu']) else: return '' diff --git a/flow/project.py b/flow/project.py index 27070cb80..50ccee990 100644 --- a/flow/project.py +++ b/flow/project.py @@ -123,6 +123,7 @@ class IgnoreConditions(IntFlag): # This operator must be defined since IntFlag simply performs an integer # bitwise not on the underlying enum value, which is problematic in # twos-complement arithmetic. What we want is to only flip valid bits. + def __invert__(self): # Compute the largest number of bits used to represent one of the flags # so that we can XOR the appropriate number. @@ -302,11 +303,11 @@ def __str__(self): def __repr__(self): return "{type}(name='{name}', jobs='{jobs}', cmd={cmd}, directives={directives})".format( - type=type(self).__name__, - name=self.name, - jobs="[" + " ,".join(map(repr, self._jobs)) + "]", - cmd=repr(self.cmd), - directives=self.directives) + type=type(self).__name__, + name=self.name, + jobs="[" + " ,".join(map(repr, self._jobs)) + "]", + cmd=repr(self.cmd), + directives=self.directives) def __hash__(self): return int(sha1(self.id.encode('utf-8')).hexdigest(), 16) @@ -387,6 +388,7 @@ class JobOperation(_JobOperation): :type directives: :class:`flow.directives._Directives` """ + def __init__(self, id, name, job, cmd, directives=None): self._id = id self.name = name @@ -421,11 +423,11 @@ def job(self): def __repr__(self): return "{type}(name='{name}', job='{job}', cmd={cmd}, directives={directives})".format( - type=type(self).__name__, - name=self.name, - job=repr(self._jobs[0]), - cmd=repr(self.cmd), - directives=self.directives) + type=type(self).__name__, + name=self.name, + job=repr(self._jobs[0]), + cmd=repr(self.cmd), + directives=self.directives) class _SubmissionJobOperation(_JobOperation): @@ -701,6 +703,7 @@ class FlowGroupEntry(object): :type options: str """ + def __init__(self, name, options=""): self.name = name self.options = options @@ -1021,11 +1024,11 @@ def _generate_id(self, jobs, operation_name=None, index=0): separator = getattr(project._environment, 'JOB_ID_SEPARATOR', '/') readable_name = '{project}{sep}{jobs}{sep}{op_string}{sep}{index:04d}{sep}'.format( - sep=separator, - project=str(project)[:12], - jobs=concat_jobs_str, - op_string=op_string[:12], - index=index)[:max_len] + sep=separator, + project=str(project)[:12], + jobs=concat_jobs_str, + op_string=op_string[:12], + index=index)[:max_len] # By appending the unique job_op_id, we ensure that each id is truly unique. return readable_name + job_op_id @@ -1714,10 +1717,10 @@ def _get_operations_status(self, job, cached_status): for operation in group.operations: if scheduler_status >= status_dict[operation]['scheduler_status']: status_dict[operation] = { - 'scheduler_status': scheduler_status, - 'eligible': eligible, - 'completed': completed - } + 'scheduler_status': scheduler_status, + 'eligible': eligible, + 'completed': completed + } for key in sorted(status_dict): yield key, status_dict[key] @@ -2633,14 +2636,6 @@ def select(operation): self._run_operations(operations, pretend=pretend, np=np, timeout=timeout, progress=progress) - def _generate_operations(self, cmd, jobs, requires=None): - "Generate job-operations for a given 'direct' command." - for job in jobs: - if requires and set(requires).difference(self.labels(job)): - continue - cmd_ = cmd.format(job=job) - yield _JobOperation(name=cmd_.replace(' ', '-'), cmd=cmd_, jobs=(job,)) - def _gather_flow_groups(self, names=None): """Grabs FlowGroups that match any of a set of names.""" operations = OrderedDict() @@ -3078,7 +3073,6 @@ def _add_script_args(cls, parser): '-p', '--parallel', action='store_true', help="Execute all operations in parallel.") - cls._add_direct_cmd_arg_group(parser) cls._add_template_arg_group(parser) @classmethod @@ -3171,22 +3165,6 @@ def _add_operation_bundling_arg_group(cls, parser): action='store_true', help="Execute all operations in a single bundle in parallel.") - @classmethod - def _add_direct_cmd_arg_group(cls, parser): - direct_cmd_group = parser.add_argument_group("direct cmd") - direct_cmd_group.add_argument( - '--cmd', - type=str, - help="Directly specify the command for an operation. " - "For example: --cmd='echo {job._id}'. " - "--cmd option is deprecated as of 0.9 and will be removed in 0.11.") - direct_cmd_group.add_argument( - '--requires', - type=str, - nargs='+', - help="Manually specify all labels that are required for the direct command " - "to be considered eligible for execution.") - def export_job_statuses(self, collection, statuses): "Export the job statuses to a database collection." for status in statuses: @@ -3276,7 +3254,7 @@ def _add_print_status_args(cls, parser): "The '--no-parallelize' argument is deprecated. " "Please use the status_parallelization configuration " "instead (see above)." - ) + ) view_group.add_argument( '-o', '--output-format', type=str, @@ -3708,28 +3686,16 @@ def _main_run(self, args): def _main_script(self, args): "Generate a script for the execution of operations." - if args.requires and not args.cmd: - raise ValueError( - "The --requires option can only be used in combination with --cmd.") - if args.cmd and args.operation_name: - raise ValueError( - "Cannot use the -o/--operation-name and the --cmd options in combination!") # Select jobs: jobs = self._select_jobs_from_args(args) # Gather all pending operations or generate them based on a direct command... with self._potentially_buffered(): - if args.cmd: - warnings.warn("The --cmd option for script is deprecated as of " - "0.10 and will be removed in 0.12.", - DeprecationWarning) - operations = self._generate_operations(args.cmd, jobs, args.requires) - else: - names = args.operation_name if args.operation_name else None - default_directives = self._get_default_directives() - operations = self._get_submission_operations(jobs, default_directives, names, - args.ignore_conditions, - args.ignore_conditions_on_execution) + names = args.operation_name if args.operation_name else None + default_directives = self._get_default_directives() + operations = self._get_submission_operations(jobs, default_directives, names, + args.ignore_conditions, + args.ignore_conditions_on_execution) operations = list(islice(operations, args.num)) # Generate the script and print to screen. diff --git a/flow/version.py b/flow/version.py index 8c824424b..04020b868 100644 --- a/flow/version.py +++ b/flow/version.py @@ -2,6 +2,6 @@ # All rights reserved. # This software is licensed under the BSD 3-Clause License. -__version__ = '0.10.0' +__version__ = '0.11.0' __all__ = ['__version__'] diff --git a/setup.cfg b/setup.cfg index eb57743e7..eefc0cc80 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.10.0 +current_version = 0.11.0 commit = True tag = False message = Bump up to version {new_version}. @@ -15,7 +15,7 @@ match = template_filters.py [flake8] max-line-length = 100 -exclude = mistune,doc/conf.py +exclude = mistune,doc/conf.py,.sync-zenodo-metadata.py [bumpversion:file:setup.py] @@ -28,6 +28,6 @@ exclude = mistune,doc/conf.py [bumpversion:file:.zenodo.json] [tool:pytest] -filterwarnings = +filterwarnings = ignore:.*get_id is deprecated.*:DeprecationWarning ignore:.*The env argument is deprecated*:DeprecationWarning diff --git a/setup.py b/setup.py index 2abc8fc49..e20bbae9c 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ setup( name='signac-flow', - version='0.10.0', + version='0.11.0', packages=find_packages(), include_package_data=True, zip_safe=True,