compile.bzl

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

load("//:plugin.bzl", "ProtoPluginInfo")

ProtoCompileInfo = provider(fields = {
    "label": "label object",
    "plugins": "ProtoPluginInfo object",
    "descriptor": "descriptor set file",
    "outputs": "generated protoc outputs",
    "files": "final generated files",
    "protos": "generated protos (copies)",
    "args": "proto arguments",
    "tools": "proto tools",
    "verbose": "verbose level",
})

def _get_plugin_out(ctx, plugin):
    if not plugin.out:
        return None
    filename = plugin.out
    filename = filename.replace("{name}", ctx.label.name)
    return filename


# From https://github.com/grpc/grpc/blob/2e7d6b94eaf6b0e11add27606b4fe3d0b7216154/bazel/protobuf.bzl:

_VIRTUAL_IMPORTS = "/_virtual_imports/"

def _strip_virtual_import(path):
      pos = path.find(_VIRTUAL_IMPORTS)
      path = path[pos + len(_VIRTUAL_IMPORTS):]
      return path.split("/", 1)[-1]

def _get_proto_filename(src):
    """Assemble the filename for a proto

    Args:
      src: the .proto <File>

    Returns:
      <string> of the filename.
    """
    parts = src.short_path.split("/")
    if len(parts) > 1 and parts[0] == "..":
        return "/".join(parts[2:])
    return src.short_path

def _apply_plugin_transitivity_rules(ctx, targets, plugin):
    """Process the proto target list according to plugin transitivity rules

    Args:
      ctx: the <ctx> object
      targets: the dict<string,File> of .proto files that we intend to compile.
      plugin: the <PluginInfo> object.

    Returns:
      <list<File>> the possibly filtered list of .proto <File>s
    """

    # Iterate transitivity rules like '{ "google/protobuf": "exclude" }'. The
    # only rule type implemented is "exclude", which checks if the pathname or
    # dirname ends with the given pattern.  If so, remove that item in the
    # targets list.
    #
    # Why does this feature exist?  Well, library rules like C# require all the
    # proto files to be present during the compilation (collected via transitive
    # sources).  However, since the well-known types are already present in the
    # library dependencies, we don't actually want to compile well-known types
    # (but do want to compile everything else).
    #
    transitivity = {}
    transitivity.update(plugin.transitivity)
    transitivity.update(ctx.attr.transitivity)

    for pattern, rule in transitivity.items():
        if rule == "exclude":
            for key, target in targets.items():
                if ctx.attr.verbose > 2:
                    print("Checking '%s' endswith '%s'" % (target.short_path, pattern))
                if target.dirname.endswith(pattern) or target.path.endswith(pattern):
                    targets.pop(key)
                    if ctx.attr.verbose > 2:
                        print("Removing '%s' from the list of files to compile as plugin '%s' excluded it" % (target.short_path, plugin.name))
                elif ctx.attr.verbose > 2:
                    print("Keeping '%s' (not excluded)" % (target.short_path))
        elif rule == "include":
            for key, target in targets.items():
                if target.dirname.endswith(pattern) or target.path.endswith(pattern):
                    if ctx.attr.verbose > 2:
                        print("Keeping '%s' (explicitly included)" % (target.short_path))
                else:
                    targets.pop(key)
                    if ctx.attr.verbose > 2:
                        print("Removing '%s' from the list of files to compile as plugin '%s' did not include it" % (target.short_path, plugin.name))
        else:
            fail("Unknown transitivity rule '%s'" % rule)
    return targets

def get_plugin_outputs(ctx, descriptor, outputs, src, proto, plugin):
    """Get the predicted generated outputs for a given plugin

    Args:
      ctx: the <ctx> object
      descriptor: the descriptor <Generated File>
      outputs: the list of outputs.
      src: the orginal .proto source <Source File>.
      proto: the copied .proto <Generated File> (the one in the package 'staging area')
      plugin: the <PluginInfo> object.

    Returns:
      <list<Generated File>> the augmented list of files that will be generated
    """
    for output in plugin.outputs:
        filename = _get_output_filename(src, plugin, output)
        if not filename:
            continue
        sibling = _get_output_sibling_file(output, proto, descriptor)
        outputs.append(ctx.actions.declare_file(filename, sibling = sibling))
    return outputs

def _get_output_filename(src, plugin, pattern):
    """Build the predicted filename for file generated by the given plugin.

    A 'proto_plugin' rule allows one to define the predicted outputs.  For
    flexibility, we allow special tokens in the output filename that get
    replaced here. The overall pattern is '{token}' mimicking the python
    'format' feature.

    Additionally, there are '|' characters like '{basename|pascal}' that can be
    read as 'take the basename and pipe that through the pascal function'.

    Args:
      src: the .proto <File>
      plugin: the <PluginInfo> object.
      pattern: the input pattern string

    Returns:
      the replaced string
    """

    # If output to srcjar, don't emit a per-proto output file.
    if plugin.out:
        return None

    # Slice off this prefix if it exists, we don't use it here.
    if pattern.startswith("{package}/"):
        pattern = pattern[len("{package}/"):]
    basename = src.basename
    if basename.endswith(".proto"):
        basename = basename[:-6]
    elif basename.endswith(".protodevel"):
        basename = basename[:-11]

    filename = basename

    if pattern.find("{basename}") != -1:
        filename = pattern.replace("{basename}", basename)
    elif pattern.find("{basename|pascal}") != -1:
        filename = pattern.replace("{basename|pascal}", _pascal_case(basename))
    elif pattern.find("{basename|pascal|objc}") != -1:
        filename = pattern.replace("{basename|pascal|objc}", _pascal_objc(basename))
    elif pattern.find("{basename|rust_keyword}") != -1:
        filename = pattern.replace("{basename|rust_keyword}", _rust_keyword(basename))
    else:
        filename = basename + pattern

    return filename

def _get_output_sibling_file(pattern, proto, descriptor):
    """Get the correct place to

    The ctx.actions.declare_file has a 'sibling = <File>' feature that allows
    one to declare files in the same directory as the sibling.

    This function checks for the prefix special token '{package}' and, if true,
    uses the descriptor as the sibling (which declares the output file will be
    in the root of the generated tree).

    Args:
      pattern: the input filename pattern <string>
      proto: the .proto <Generated File> (in the staging area)
      descriptor: the descriptor <File> that marks the staging root.

    Returns:
      the <File> to be used as the correct sibling.
    """

    if pattern.startswith("{package}/"):
        return descriptor
    return proto

rust_keywords = {
    "as": True,
    "break": True,
    "const": True,
    "continue": True,
    "crate": True,
    "else": True,
    "enum": True,
    "extern": True,
    "false": True,
    "fn": True,
    "for": True,
    "if": True,
    "impl": True,
    "let": True,
    "loop": True,
    "match": True,
    "mod": True,
    "move": True,
    "mut": True,
    "pub": True,
    "ref": True,
    "return": True,
    "self": True,
    "Self": True,
    "static": True,
    "struct": True,
    "super": True,
    "trait": True,
    "true": True,
    "type": True,
    "unsafe": True,
    "use": True,
    "where": True,
    "while": True,
}

objc_upper_segments = {
    "url": "URL",
    "http": "HTTP",
    "https": "HTTPS",
}

def _capitalize(s):
    """Capitalize a string - only first letter
    Args:
      s (string): The input string to be capitalized.
    Returns:
      (string): The capitalized string.
    """
    return s[0:1].upper() + s[1:]

def get_plugin_out_arg(ctx, outdir, plugin, plugin_outfiles):
    """Build the --java_out argument

    Args:
      ctx: the <ctx> object
      outdir: the package output directory <string>
      plugin: the <PluginInfo> object.
      plugin_outfiles: The <dict<string,<File>>.  For example, {closure: "library.js"}

    Returns
      <string> for the protoc arg list.
    """

    arg = outdir
    if plugin.outdir:
        arg = plugin.outdir.replace("{name}", outdir)
    elif plugin.out:
        outfile = plugin_outfiles[plugin.name]
        arg = outfile.path

    # Collate a list of options from the plugin itself PLUS options from the
    # global plugin_options list (if they exist)
    options = []
    options += getattr(plugin, "options", [])
    options += getattr(ctx.attr, "plugin_options", [])

    if options:
        arg = "%s:%s" % (",".join(_get_plugin_options(ctx, options)), arg)
    return "--%s_out=%s" % (plugin.name, arg)

def _get_plugin_options(ctx, options):
    """Build a plugin option list

    Args:
      ctx: the <ctx> object
      options: list<string> options from the <PluginInfo>

    Returns:
      <string> for the --plugin_out= arg
    """
    return [_get_plugin_option(ctx, option) for option in options]

def _get_plugin_option(ctx, option):
    """Build a plugin option

    Args:
      ctx: the <ctx> object
      option: string from the <PluginInfo>

    Returns:
      <string> for the --plugin_out= arg
    """
    return option.replace("{name}", ctx.label.name)

def _rust_keyword(s):
    """Check if arg is a rust keyword and append '_pb' if true.
    Args:
        s (string): The input string to be capitalized.
    Returns:
        (string): The appended string.
    """
    return s + "_pb" if rust_keywords.get(s) else s

def _pascal_objc(s):
    """Convert pascal_case -> PascalCase

    Objective C uses pascal case, but there are e exceptions that it uppercases
    the entire segment: url, http, and https.

    https://github.com/protocolbuffers/protobuf/blob/54176b26a9be6c9903b375596b778f51f5947921/src/google/protobuf/compiler/objectivec/objectivec_helpers.cc#L91

    Args:
      s (string): The input string to be capitalized.
    Returns: (string): The capitalized string.
    """
    segments = []
    for segment in s.split("_"):
        repl = objc_upper_segments.get(segment)
        if repl:
            segment = repl
        else:
            segment = _capitalize(segment)
        segments.append(segment)
    return "".join(segments)

def _pascal_case(s):
    """Convert pascal_case -> PascalCase
    Args:
        s (string): The input string to be capitalized.
    Returns:
        (string): The capitalized string.
    """
    return "".join([_capitalize(part) for part in s.split("_")])

def is_in_virtual_imports(source_file, virtual_folder = _VIRTUAL_IMPORTS):
    """Determines if source_file is virtual (is placed in _virtual_imports
    subdirectory). The output of all proto_library targets which use
    import_prefix  and/or strip_import_prefix arguments is placed under
    _virtual_imports directory.
    Args:
        source_file: A proto file.
        virtual_folder: The virtual folder name (is set to "_virtual_imports"
            by default)
    Returns:
        True if source_file is located under _virtual_imports, False otherwise.
    """
    return not source_file.is_source and virtual_folder in source_file.path

def copy_proto(ctx, descriptor, src):
    """Copy a proto to the 'staging area'

    Args:
      ctx: the <ctx> object
      descriptor: the descriptor <File> that marks the root of the 'staging area'.
      src: the source .proto <File>

    Returns:
      <Generated File> for the copied .proto
    """
    if is_in_virtual_imports(src):
        proto_rpath = _strip_virtual_import(src.path)
    else:
        proto_rpath = _get_proto_filename(src)
    proto_copy_path = "/".join([descriptor.dirname, proto_rpath])
    proto = ctx.actions.declare_file(proto_rpath, sibling = descriptor)

    ctx.actions.run_shell(
        mnemonic = "CopyProto",
        inputs = [src],
        outputs = [proto],
        command = "cp %s %s" % (src.path, proto_copy_path),
    )
    return proto

def _copy_jar_to_srcjar(ctx, jar):
    """Copy .jar to .srcjar

    Args:
      ctx: the <ctx> object
      jar: the <Generated File> of a jar containing source files.

    Returns:
      <Generated File> for the renamed file
    """
    srcjar = ctx.actions.declare_file("%s/%s.srcjar" % (ctx.label.name, ctx.label.name))
    ctx.actions.run_shell(
        mnemonic = "CopySrcjar",
        inputs = [jar],
        outputs = [srcjar],
        command = "mv %s %s" % (jar.path, srcjar.path),
    )
    return srcjar

def get_plugin_runfiles(tool):
    """Gather runfiles for a plugin.
    """
    files = []
    if not tool:
        return files

    info = tool[DefaultInfo]
    if not info:
        return files

    if info.files:
        files += info.files.to_list()

    if info.default_runfiles:
        runfiles = info.default_runfiles
        if runfiles.files:
            files += runfiles.files.to_list()

    if info.data_runfiles:
        runfiles = info.data_runfiles
        if runfiles.files:
            files += runfiles.files.to_list()

    return files

def proto_compile_impl(ctx):
    ###
    ### Part 1: setup variables used in scope
    ###

    # <int> verbose level
    verbose = ctx.attr.verbose

    # <File> the protoc tool
    protoc = ctx.executable.protoc

    # <File> for the output descriptor.  Often used as the sibling in
    # 'declare_file' actions.
    descriptor = ctx.outputs.descriptor

    # <string> The directory where that generated descriptor is.
    outdir = descriptor.dirname

    # <list<ProtoInfo>> A list of ProtoInfo
    deps = [dep[ProtoInfo] for dep in ctx.attr.deps]

    # <list<PluginInfo>> A list of PluginInfo
    plugins = [plugin[ProtoPluginInfo] for plugin in ctx.attr.plugins]

    # <list<File>> The list of .proto files that will exist in the 'staging
    # area'.  We copy them from their source location into place such that a
    # single '-I.' at the package root will satisfy all import paths.
    protos = []

    # <dict<string,File>> The set of .proto files to compile, used as the final
    # list of arguments to protoc.  This is a subset of the 'protos' list that
    # are directly specified in the proto_library deps, but excluding other
    # transitive .protos.  For example, even though we might transitively depend
    # on 'google/protobuf/any.proto', we don't necessarily want to actually
    # generate artifacts for it when compiling 'foo.proto'. Maintained as a dict
    # for set semantics.  The key is the value from File.path.
    targets = {}

    # <dict<string,File>> A mapping from plugin name to the plugin tool. Used to
    # generate the --plugin=protoc-gen-KEY=VALUE args
    plugin_tools = {}

    # <dict<string,<File> A mapping from PluginInfo.name to File.  In the case
    # of plugins that specify a single output 'archive' (like java), we gather
    # them in this dict.  It is used to generate args like
    # '--java_out=libjava.jar'.
    plugin_outfiles = {}

    # <list<File>> The list of srcjars that we're generating (like
    # 'foo.srcjar').
    srcjars = []

    # <list<File>> The list of generated artifacts like 'foo_pb2.py' that we
    # expect to be produced.
    outputs = []

    # Additional data files from plugin.data needed by plugin tools that are not
    # single binaries.
    data = []

    ###
    ### Part 2: gather plugin.out artifacts
    ###

    # Some protoc plugins generate a set of output files (like python) while
    # others generate a single 'archive' file that contains the individual
    # outputs (like java).  This first loop is for the latter type.  In this
    # scenario, the PluginInfo.out attribute will exist; the predicted file
    # output location is relative to the package root, marked by the descriptor
    # file. Jar outputs are gathered as a special case as we need to
    # post-process them to have a 'srcjar' extension (java_library rules don't
    # accept source jars with a 'jar' extension)
    for plugin in plugins:
        if plugin.executable:
            plugin_tools[plugin.name] = plugin.executable
        data += plugin.data + get_plugin_runfiles(plugin.tool)

        filename = _get_plugin_out(ctx, plugin)
        if not filename:
            continue
        out = ctx.actions.declare_file(filename, sibling = descriptor)
        outputs.append(out)
        plugin_outfiles[plugin.name] = out
        if out.path.endswith(".jar"):
            srcjar = _copy_jar_to_srcjar(ctx, out)
            srcjars.append(srcjar)

    ###
    ### Part 3a: Gather generated artifacts for each dependency .proto source file.
    ###

    for dep in deps:
        # Iterate all the directly specified .proto files.  If we have already
        # processed this one, skip it to avoid declaring duplicate outputs.
        # Create an action to copy the proto into our staging area.  Consult the
        # plugin to assemble the actual list of predicted generated artifacts
        # and save these in the 'outputs' list.
        for src in dep.direct_sources:
            if targets.get(src.path):
                continue
            proto = copy_proto(ctx, descriptor, src)
            targets[src] = proto
            protos.append(proto)

        # Iterate all transitive .proto files.  If we already processed in the
        # loop above, skip it. Otherwise add a copy action to get it into the
        # 'staging area'
        for src in dep.transitive_sources.to_list():
            if targets.get(src):
                continue
            if verbose > 2:
                print("transitive source: %r" % src)
            proto = copy_proto(ctx, descriptor, src)
            protos.append(proto)
            if ctx.attr.transitive:
                targets[src] = proto

    ###
    ### Part 3b: apply transitivity rules
    ###

    # If the 'transitive = true' was enabled, we collected all the protos into
    # the 'targets' list.
    # At this point we want to post-process that list and remove any protos that
    # might be incompatible with the plugin transitivity rules.
    if ctx.attr.transitive:
        for plugin in plugins:
            targets = _apply_plugin_transitivity_rules(ctx, targets, plugin)

    ###
    ### Part 3c: collect generated artifacts for all in the target list of protos to compile
    ###
    for src, proto in targets.items():
        for plugin in plugins:
            outputs = get_plugin_outputs(ctx, descriptor, outputs, src, proto, plugin)

    ###
    ### Part 4: build list of arguments for protoc
    ###

    args = ["--descriptor_set_out=%s" % descriptor.path]

    # By default we have a single 'proto_path' argument at the 'staging area'
    # root.
    args += ["--proto_path=%s" % outdir]

    if ctx.attr.include_imports:
        args += ["--include_imports"]

    if ctx.attr.include_source_info:
        args += ["--include_source_info"]

    for plugin in plugins:
        args += [get_plugin_out_arg(ctx, outdir, plugin, plugin_outfiles)]

    args += ["--plugin=protoc-gen-%s=%s" % (k, v.path) for k, v in plugin_tools.items()]

    args += [proto.path for proto in targets.values()]

    ###
    ### Part 5: build the final protoc command and declare the action
    ###

    mnemonic = "ProtoCompile"

    command = " ".join([protoc.path] + args)

    if verbose > 0:
        print("%s: %s" % (mnemonic, command))
    if verbose > 1:
        command += " && echo '\n##### SANDBOX AFTER RUNNING PROTOC' && find ."
    if verbose > 2:
        command = "echo '\n##### SANDBOX BEFORE RUNNING PROTOC' && find . && " + command
    if verbose > 3:
        command = "env && " + command
        for f in outputs:
            print("expected output: ", f.path)


    tools = [protoc] + plugin_tools.values()
    inputs = protos + data
    outs = outputs + [descriptor] + ctx.outputs.outputs

    if verbose > 3:
        for s in args:
            print("ARG: %s" % s)
        for k, f in targets.items():
            print("TARGET: %s=%s" % (k, f))
        for f in tools:
            print("TOOL: %s" % f.path)
        for f in inputs:
            print("INPUT: %s" % f.path)
        for f in outs:
            print("OUTPUT: %s" % f.path)

    ctx.actions.run_shell(
        mnemonic = mnemonic,
        command = command,
        inputs = inputs,
        outputs = outs,
        tools = tools,
    )

    ###
    ### Part 6: assemble output providers
    ###

    # The files for 'DefaultInfo' include any explicit outputs for the rule.  If
    # we are generating srcjars, use those as the final outputs rather than
    # their '.jar' intermediates.  Otherwise include all the file outputs.
    # NOTE: this looks a little wonky here.  It probably works in simple cases
    # where there list of plugins has length 1 OR all outputting to jars OR all
    # not outputting to jars.  Probably would break here if they were mixed.
    files = [] + ctx.outputs.outputs

    if len(srcjars) > 0:
        files += srcjars
    else:
        files += outputs
        if len(plugin_outfiles) > 0:
            files += plugin_outfiles.values()

    return [ProtoCompileInfo(
        label = ctx.label,
        plugins = plugins,
        protos = protos,
        outputs = outputs,
        files = files,
        tools = plugin_tools,
        args = args,
        descriptor = descriptor,
    ), DefaultInfo(files = depset(files))]


proto_compile = rule(
    implementation = proto_compile_impl,
    attrs = {
        "deps": attr.label_list(
            doc = "proto_library dependencies",
            mandatory = True,
            providers = [ProtoInfo],
        ),
        "plugins": attr.label_list(
            doc = "List of protoc plugins to apply",
            providers = [ProtoPluginInfo],
            mandatory = True,
        ),
        "plugin_options": attr.string_list(
            doc = "List of additional 'global' options to add (applies to all plugins)",
        ),
        "outputs": attr.output_list(
            doc = "Escape mechanism to explicitly declare files that will be generated",
        ),
        "protoc": attr.label(
            doc = "The protoc tool",
            default = "@com_google_protobuf//:protoc",
            cfg = "host",
            executable = True,
        ),
        "verbose": attr.int(
            doc = "Increase verbose level for more debugging",
        ),
        "include_imports": attr.bool(
            doc = "Pass the --include_imports argument to the protoc_plugin",
            default = True,
        ),
        "include_source_info": attr.bool(
            doc = "Pass the --include_source_info argument to the protoc_plugin",
            default = True,
        ),
        "transitive": attr.bool(
            doc = "Emit transitive artifacts",
        ),
        "transitivity": attr.string_dict(
            doc = "Transitive rules.  When the 'transitive' property is enabled, this string_dict can be used to exclude protos from the compilation list",
        ),
    },
    # TODO(pcj) remove this
    outputs = {
        "descriptor": "%{name}/descriptor.source.bin",
    },
    output_to_genfiles = True,
)