Implement resource merging for flavored source sets - Part 2 (#165)

Fixes #165

.bazelrc

@@ -1,30 +1,40 @@
 # BzlMod
 common --enable_bzlmod=false
 
-# Cache
+# Cache setup
 common --disk_cache=bazel-cache
 common --remote_cache_compression=true
+common --remote_upload_local_results=false # Don't upload results by default
 startup --digest_function=blake3
-common --experimental_action_cache_store_output_metadata
+common --experimental_remote_cache_eviction_retries=10
 common --modify_execution_info=JavaDeployJar=+no-remote-cache # Don't cache Java deploy jar which is huge in size
+common --remote_cache_compression=true
+common --experimental_disk_cache_gc_max_size=4G
 
-# Errors
+# Error config
 common --verbose_failures
 
-# Action Env
+# Env config
 common --incompatible_strict_action_env
 common --reuse_sandbox_directories
 common --repo_env=RJE_VERBOSE=true
 
+# JVM External rules
+common --@rules_jvm_external//settings:stamp_manifest=False
+
+# Resources
+common --local_resources=cpu=HOST_CPUS*.75 # Android actions start their own threads which can overwhelm the system
+
 # JAVA - START
 common --experimental_strict_java_deps=off # Turn off strict java deps
 common --java_runtime_version=remotejdk_17 # Use inbuilt Java 17 for hermeticity
 common --tool_java_runtime_version=remotejdk_17
 common --tool_java_language_version=17
 common --java_language_version=17
+## See https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/Locale.html#legacy_language_codes
 common --jvmopt="-Djava.locale.providers=COMPAT,SPI" # Use Java 8 default locale provider
 common --jvmopt="--add-exports=java.xml/com.sun.org.apache.xerces.internal.dom=ALL-UNNAMED"
-common --experimental_java_classpath=bazel 
+common --experimental_java_classpath=bazel
 common --experimental_java_header_input_pruning
 # JAVA - END
 
@@ -46,37 +56,60 @@ common --output_library_merged_assets=false # Turn off asset merging artifact
 
 # Workers
 common --worker_verbose
-common --experimental_worker_multiplex
+common --worker_multiplex
 common --experimental_shrink_worker_pool
 common --experimental_worker_for_repo_fetching=platform
+common --experimental_collect_worker_data_in_profiler
 
+## Android Resource Workers
 common --experimental_persistent_aar_extractor
-common --persistent_multiplex_android_tools
 common --persistent_android_dex_desugar
+common --persistent_android_resource_processor
+common --persistent_android_dex_desugar
+common --persistent_multiplex_android_dex_desugar
+common --persistent_multiplex_android_resource_processor
+common --persistent_multiplex_android_tools
 
+# Action Strategies
+common --strategy=AARGenerator=worker
 common --strategy=DatabindingStubs=worker
-common --worker_max_instances=Javac=1
-common --worker_max_instances=KotlinCompile=1
-common --worker_max_instances=KotlinKapt=1
+common --modify_execution_info=GenerateDataBindingBaseClasses=+supports-multiplex-workers=1
+## Java Workers
+common --strategy=KotlinCompile=worker
+common --strategy=Javac=worker
+# common --strategy=Turbine=worker
+## Worker configuration to avoid CPU thrashing https://github.com/bazelbuild/bazel/issues/8586#issuecomment-500070549
+common --worker_max_instances=Aapt2Optimize=1
 common --worker_max_instances=AaptPackage=1
-common --worker_max_instances=AndroidResourceParser=1
+common --worker_max_instances=AndroidAapt2=1
+common --worker_max_instances=AndroidAssetMerger=1
+common --worker_max_instances=AndroidCompiledResourceMerger=1
 common --worker_max_instances=AndroidResourceCompiler=1
+common --worker_max_instances=AndroidResourceMerger=1
+common --worker_max_instances=AndroidResourceParser=1
 common --worker_max_instances=AndroidResourceValidator=1
 common --worker_max_instances=AndroidLintAnalyze=1
 common --worker_max_instances=AndroidLint=1
-common --worker_max_instances=RClassGenerator=1
-common --worker_max_instances=AndroidAapt2=1
-common --worker_max_instances=AndroidAssetMerger=1
-common --worker_max_instances=AndroidResourceMerger=1
-common --worker_max_instances=AndroidCompiledResourceMerger=1
-common --worker_max_instances=Aapt2Optimize=1
+common --worker_max_instances=BuildConfigGenerationWorker=1
 common --worker_max_instances=DatabindingStubs=1
-common --worker_max_instances=GenerateDataBindingBaseClasses=1
-common --worker_max_instances=DexBuilder=1
+common --worker_max_instances=DatabindingWorker=1
 common --worker_max_instances=Desugar=1
+common --worker_max_instances=DexBuilder=1
+common --worker_max_instances=GenerateDataBindingBaseClasses=1
+common --worker_max_instances=Javac=1
+common --worker_max_instances=JdepsMerge=1
+common --worker_max_instances=KotlinCompile=1
+common --worker_max_instances=KotlinKapt=1
+common --worker_max_instances=MergeSourceSets=1
+common --worker_max_instances=ProcessDatabinding=1
+common --worker_max_instances=RClassGenerator=1
+common --worker_max_instances=Turbine=1
 
-test --test_output=errors # Print test logs for failed tests
+# Test config
 test --build_tests_only
+test --test_verbose_timeout_warnings
+test --test_output=errors # Print test logs for failed tests
+test --test_summary=terse # Print information only about unsuccessful tests that were run
 
 run --ui_event_filters=-info,-stdout --noshow_progress
 

.bazelversion

@@ -1 +1 @@
-7.2.1
+7.4.0

rules/android/android_binary.bzl

@@ -1,15 +1,15 @@
-load("@grab_bazel_common//tools/build_config:build_config.bzl", _build_config = "build_config")
-load("@grab_bazel_common//tools/kotlin:android.bzl", "kt_android_library")
 load("@grab_bazel_common//rules/android/databinding:databinding.bzl", "DATABINDING_DEPS")
 load("@grab_bazel_common//rules/android/lint:defs.bzl", "LINT_ENABLED", "lint", "lint_sources", _lint_baseline = "baseline")
 load("@grab_bazel_common//rules/check/detekt:defs.bzl", "detekt")
+load("@grab_bazel_common//tools/build_config:build_config.bzl", _build_config = "build_config")
+load("@grab_bazel_common//tools/kotlin:android.bzl", "kt_android_library")
 load(":resources.bzl", "build_resources")
 
 def android_binary(
         name,
         debug = True,
         build_config = {},
-        custom_package = {},
+        custom_package = "",
         res_values = {},
         enable_data_binding = False,
         enable_compose = False,
@@ -44,13 +44,17 @@ def android_binary(
         longs = build_config.get("longs", default = {}),
         strings = build_config.get("strings", default = {}),
     )
-
-    resource_files = build_resources(
+    merged_resources = build_resources(
         name = name,
+        is_binary = True,
+        namespace = attrs.get("manifest_values")["applicationId"],
+        manifest = attrs.get("manifest", None),
         resource_files = attrs.get("resource_files", default = []),
-        resources = attrs.get("resources", default = {}),
+        resource_sets = attrs.get("resource_sets", default = {}),
         res_values = res_values,
     )
+    resource_files = merged_resources.res
+    manifest = merged_resources.manifest
 
     # Kotlin compilation with kt_android_library
     kotlin_target = "lib_" + name
@@ -61,10 +65,10 @@ def android_binary(
     kt_android_library(
         name = kotlin_target,
         srcs = attrs.get("srcs", default = []),
-        assets = attrs.get("assets", default = None),
-        assets_dir = attrs.get("assets_dir", default = None),
+        assets = merged_resources.assets,
+        assets_dir = merged_resources.asset_dir,
         custom_package = custom_package,
-        manifest = attrs.get("manifest", default = None),
+        manifest = manifest,
         resource_files = resource_files,
         visibility = attrs.get("visibility", default = None),
         deps = kotlin_library_deps,
@@ -73,14 +77,15 @@ def android_binary(
     lint_enabled = lint_options.get("enabled", False) and (len(attrs.get("srcs", default = [])) > 0 or len(resource_files) > 0)
     tags = []
     android_binary_deps = [kotlin_target]
+
     if lint_enabled:
         lint_sources_target = "_" + name + "_lint_sources"
         lint_baseline = _lint_baseline(lint_options.get("baseline", None))
         lint_sources(
             name = lint_sources_target,
             srcs = attrs.get("srcs", default = []),
             resources = [file for file in resource_files if file.endswith(".xml")],
-            manifest = attrs.get("manifest"),
+            manifest = manifest,
             baseline = lint_baseline,
             lint_config = lint_options.get("config", None),
             deps = kotlin_library_deps,
@@ -127,7 +132,7 @@ def android_binary(
         dexopts = attrs.get("dexopts", default = None),
         incremental_dexing = attrs.get("incremental_dexing", default = None),
         javacopts = attrs.get("javacopts", default = None),
-        manifest = attrs.get("manifest"),
+        manifest = manifest,
         multidex = attrs.get("multidex", default = None),
         manifest_values = attrs.get("manifest_values", default = None),
         resource_configuration_filters = attrs.get("resource_configuration_filters", default = None),

rules/android/android_library.bzl

@@ -1,16 +1,16 @@
-load("@grab_bazel_common//tools/build_config:build_config.bzl", _build_config = "build_config")
-load("@grab_bazel_common//tools/kotlin:android.bzl", "kt_android_library")
 load("@grab_bazel_common//rules/android/databinding:databinding.bzl", "kt_db_android_library")
-load(":resources.bzl", "build_resources")
 load("@grab_bazel_common//rules/android/lint:defs.bzl", "LINT_ENABLED", "lint", "lint_sources")
 load("@grab_bazel_common//rules/check/detekt:defs.bzl", "detekt")
+load("@grab_bazel_common//tools/build_config:build_config.bzl", _build_config = "build_config")
+load("@grab_bazel_common//tools/kotlin:android.bzl", "kt_android_library")
+load(":resources.bzl", "build_resources")
 
 def android_library(
         name,
         debug = True,
         srcs = [],
         build_config = {},
-        custom_package = {},
+        custom_package = "",
         res_values = {},
         enable_data_binding = False,
         enable_compose = False,
@@ -46,24 +46,30 @@ def android_library(
         strings = build_config.get("strings", default = {}),
     )
 
-    resource_files = build_resources(
+    merged_resources = build_resources(
         name = name,
+        is_binary = False,
+        namespace = custom_package,
+        manifest = attrs.get("manifest", None),
         resource_files = attrs.get("resource_files", default = []),
-        resources = attrs.get("resources", default = {}),
+        resource_sets = attrs.get("resource_sets", default = {}),
         res_values = res_values,
     )
+    resource_files = merged_resources.res
+    manifest = merged_resources.manifest
 
     lint_enabled = lint_options.get("enabled", False) and (len(srcs) > 0 or len(resource_files) > 0)
     android_library_deps = attrs.get("deps", default = []) + [build_config_target]
     tags = attrs.get("tags", default = [])
+
     if lint_enabled:
         lint_sources_target = "_" + name + "_lint_sources"
         lint_baseline = lint_options.get("baseline", None)
         lint_sources(
             name = lint_sources_target,
             srcs = srcs,
             resources = [file for file in resource_files if file.endswith(".xml")],
-            manifest = attrs.get("manifest"),
+            manifest = manifest,
             baseline = lint_baseline,
             lint_config = lint_options.get("config", None),
             deps = android_library_deps,
@@ -115,10 +121,10 @@ def android_library(
         name = name,
         srcs = srcs,
         custom_package = custom_package,
-        manifest = attrs.get("manifest"),
+        manifest = manifest,
         resource_files = resource_files,
-        assets = attrs.get("assets", default = None),
-        assets_dir = attrs.get("assets_dir", default = None),
+        assets = merged_resources.assets,
+        assets_dir = merged_resources.asset_dir,
         visibility = attrs.get("visibility", default = None),
         tags = tags,
         deps = android_library_deps,

rules/android/private/resource_merger.bzl

@@ -1,19 +1,23 @@
+load("@grab_bazel_common//rules/android:utils.bzl", "utils")
+
 """
-Rule to merge android variant specific resource folders and account for overrides.
+Rule to merge android variant specific resource folders, assets and manifest and account for overrides.
 """
 
-def _to_path(f):
-    return f.path
-
 def _resource_merger_impl(ctx):
     outputs = ctx.outputs.merged_resources
+    merged_manifest = ctx.outputs.merged_manifest
 
     # Args for compiler
     args = ctx.actions.args()
     args.set_param_file_format("multiline")
     args.use_param_file("--flagfile=%s", use_always = True)
     args.add("RESOURCE_MERGER")
+    args.add("--label", ctx.label)
+    if ctx.attr.is_binary:
+        args.add("--is-binary")
     args.add("--target", ctx.label.package)
+    args.add("--package-name", ctx.attr.namespace)
     args.add_joined(
         "--source-sets",
         ctx.attr.source_sets,
@@ -23,17 +27,18 @@ def _resource_merger_impl(ctx):
         "--output",
         outputs,
         join_with = ",",
-        map_each = _to_path,
+        map_each = utils.to_path,
     )
+    args.add("--manifest", merged_manifest)
 
     mnemonic = "MergeSourceSets"
     ctx.actions.run(
         mnemonic = mnemonic,
         inputs = depset(ctx.files.resources + ctx.files.manifests),
-        outputs = outputs,
+        outputs = ctx.outputs.merged_resources + [merged_manifest],
         executable = ctx.executable._compiler,
         arguments = [args],
-        progress_message = "%s %s" % (mnemonic, ctx.label),
+        progress_message = "%s %s" % (mnemonic, str(ctx.label).lstrip("@")),
         execution_requirements = {
             "supports-workers": "1",
             "supports-multiplex-workers": "1",
@@ -47,9 +52,12 @@ def _resource_merger_impl(ctx):
 resource_merger = rule(
     implementation = _resource_merger_impl,
     attrs = {
+        "is_binary": attr.bool(mandatory = True, default = False),
         "source_sets": attr.string_list(),
+        "namespace": attr.string(mandatory = True),
         "resources": attr.label_list(allow_files = True, mandatory = True),
         "manifests": attr.label_list(allow_files = True, mandatory = True),
+        "merged_manifest": attr.output(mandatory = True),
         "merged_resources": attr.output_list(mandatory = True),
         "_compiler": attr.label(
             default = Label("@grab_bazel_common//tools/aapt_lite:aapt_lite"),