pants.ini

# Pants config for Foursquare Fsq.io

[GLOBAL]
print_exception_stacktrace: True
enable_pantsd: False

# We use os.environ in a few RPM BUILD files that want to consume the shared version from env.sh.
# TODO(mateo): Consider a macro for environmental variables upstream.
build_file_imports: allow
kill_nailguns: True
# cc-shapefile hosting for Fsq.io users.
binaries_baseurls: +[
    'https://github.com/foursquare/cc-shapefiles/raw/master/blobs/',
  ]

# The pants_version value is only used in Fsq.io, although checked on each run everywhere.
# TODO(mateo): Move just the version declaration to a deployed file so we can unit test the
# rest of this config against prod.
pants_version: 1.7.0

backend_packages: +[
    'pants.contrib.go',
    'pants.contrib.node',
    'fsqio.pants.buildgen.core',
    'fsqio.pants.buildgen.jvm',
    'fsqio.pants.buildgen.python',
    'fsqio.pants.buildgen.spindle',
    'fsqio.pants.export',
    'fsqio.pants.ivy',
    'fsqio.pants.mypy',
    'fsqio.pants.node',
    'fsqio.pants.publish',
    'fsqio.pants.rpmbuild',
    'fsqio.pants.spindle',
    'fsqio.pants.tags',
  ]

# Mixed into all cache keys. Bump this to invalidate all existing artifacts.
# We are using timestamps for cache keys. To bump it run `date +%Y-%m-%d:%H:%M:%S` and paste in the output.
cache_key_gen_version: '2018-09-05:21:44:04'

# The build_ignore_patterns uses .gitignore syntax.
build_ignore: [
    '.*/',
    '/.pants.d',
    '.git',
    '*.orig',
    '*.rej',
    '*.pyc',
    '.local_artifact_cache',
    '.pants.bootstrap',
    '.pants.d',
    '.pantsenv',
    '.pids',
    '.pvenvs',
    '.wheelhouse',
    'environ.sh',
    '/api',
    '/dependencies',
    '/dist',
    '/etc',
    '/log',
    '/mongo-testdb',
    '/out',
    '/scripts',
    '/target',
    '/yourkit',
    'node_modules/',
    'python_cache/',
  ]


[DEFAULT]

pants_workdir: %(buildroot)s/.pants.d
pants_supportdir: %(buildroot)s/build-support
pants_distdir: %(buildroot)s/dist

_base_jvm_options: [
  '-Duser.timezone=UTC',
  '-Dfile.encoding=UTF-8',
  '-Djava.awt.headless=true',
  '-XX:+AggressiveOpts',
  '-XX:+UseCompressedOops',
  '-Xss4096k',
  '-server',
  ]

# Effectively runtime options.
_dev_jvm_options: %(_base_jvm_options)s + [
  '-XX:ReservedCodeCacheSize=512m',
  '-noverify',
  '-Dmongo.connectionsPerHost=10',
  '-Dmongo.threadsAllowedToBlockForConnectionMultiplier=10',
  ]

###
#
# Default Values for Pants Options
# These redefine the default value for all Pants options with the same name, regardles of scope.
#
###
# NOTE: These are matched by option *name* and it is easy to accidentally override more than you expected!

# Overrides *all* options named "option" - this was the only effective way to override the defaults.
jvm_options: %(_base_jvm_options)s + ['-Xmx2g']


#### Pants Task Options

[scala-platform]
version: custom
suffix_version: 2.11


[jvm-distributions]
minimum_version: 1.8.0
maximum_version: 1.8.999


[jvm-platform]
default_platform: java8
platforms: {
    'java8': {'source': '8', 'target': '8', 'args': [] },
  }


[jvm]
options: %(_base_jvm_options)s + ['-Xmx2g']

[jvm.run.jvm]
options: %(_dev_jvm_options)s + [
  '-Xmx4g',
  ]


[jvm.repl.scala]
options: %(_dev_jvm_options)s + [
  '-Xmx4g',
  '-Dscala.usejavacp=true',
  '-DconsoleMode=true',
  ]


[jvm.test.junit]
options: %(_dev_jvm_options)s + [
  "-Xmx4g",
  "-Drun.mode=test",
  '-Dscala.usejavacp=true',
  "-Djava.security.manager",
  "-Djava.security.policy=%(buildroot)s/test/jvm/fsqio.tests.policy",
  ]


[compile.zinc]
worker_count: 4
# Unneeded since we use buildgen.
unused_deps: ignore
jvm_options: %(_base_jvm_options)s + [
  '-XX:+UseConcMarkSweepGC',
  '-XX:ParallelGCThreads=4',
  '-XX:+PrintGCDetails',
  '-XX:+PrintGCTimeStamps',
  '-Dzinc.analysis.cache.limit=5000',
  '-Xmx6g',
  ]

warning_args: []
no_warning_args: [
    '-S-nowarn',
  ]
args: [
    '-S-encoding', '-SUTF-8',
    '-S-deprecation',
    '-S-unchecked',
    '-S-feature',
    '-S-Xfatal-warnings',
    '-S-g:vars',
    '-S-language:existentials',
    '-S-language:implicitConversions',
    '-S-language:reflectiveCalls',
    '-S-language:postfixOps',
    '-S-language:higherKinds',
  ]
incremental: True
fail_fast: True


[repl.scala]
main: ammonite.Main


[binary.dup]
skip: True


[test.junit]
parallel_threads: 4
use_experimental_runner: True
output_mode: FAILURE_ONLY
default_concurrency: PARALLEL_CLASSES_AND_METHODS
use_experimental_runner: True
fail_fast: True

chroot: True
fast: False


[clean-all]
async: True


[reporting]
# This cuts down on the verbosity of pants output
# Comment out these lines to see warnings and other output
console_label_format: {
    'COMPILER' : 'SUPPRESS',
  }
console_tool_output_format: {
    'COMPILER' : 'SUPPRESS',
  }


[bundle.jvm]
use_basename_prefix: True


[go-distribution]
version: 1.10.3


[thrift.gen.go-thrift]
version: 0.9.3


[goals]
logdir: /tmp/%(user)s/goals


[ivy]
# NOTE(mateo): These settings are also hardcoded in a python mixin, a hack we needed in order to test Pants tasks
# that invoke JVM tools. Changes to Ivy config often requires migrating the mixin, look under test/python/**/pants.
ivy_profile: %(pants_supportdir)s/ivy/fsqio/fsqio.ivy.xml
ivy_settings: %(pants_supportdir)s/ivy/fsqio/fsqio.ivysettings.xml
cache_dir: %(homedir)s/.pom2


[thrift]
version: 0.9.1


[node-distribution]
version: v8.8.1


[build-spindle]
jvm_options: %(_base_jvm_options)s + ['-Xmx4g']
spindle_codegen_binary: src/jvm/io/fsq/spindle/codegen:spindle


[gen.spindle]
jvm_options: %(_base_jvm_options)s + ['-Xmx4g']

scala_ssp_template: src/resources/io/fsq/ssp/codegen/scala
java_ssp_template: src/resources/io/fsq/ssp/codegen/javagen
spindle_codegen_binary: src/jvm/io/fsq/spindle/codegen:spindle
runtime_dependency: [
    '3rdparty:finagle',
    '3rdparty:thrift',
    'src/jvm/io/fsq/spindle/runtime',
  ]
write_annotations_json: true


[ide-gen.spindle-stubs]
jvm_options: %(_base_jvm_options)s + ['-Xmx4g']
scala_ssp_template: src/resources/io/fsq/ssp/codegen/scalainterface
spindle_codegen_binary: src/jvm/io/fsq/spindle/codegen:spindle
runtime_dependency: [
    '3rdparty:finagle',
    '3rdparty:thrift',
    'src/jvm/io/fsq/spindle/runtime',
  ]
# NOTE(mateo): This is the Fsq.io path, should work with internal users to migrate to this and deprecate surplus path.
stub_output_path: %(buildroot)s/scripts/fsqio/IntelliJ/spindle_stubs/


[lint.scalafmt]
skip=True


[fmt.scalafmt]
configuration: %(buildroot)s/build-support/fsqio/scalafmt/config


[fmt.scalafix]
skip=True


[pytest]
# NOTE(mateo): Keep this in sync with the version in requirements.txt.
requirements: pytest==3.6.4


[test.pytest]
timeouts: true
fail_fast: True
# Need this timeout for some luigi auto tests.
timeout_default: 110
# Overridden here because we define a jvm-related 'options' in our default section.
options=[]
chroot: True
fast: False


[jar-dependency-management]
default_target: 3rdparty:fsqio-management


[publish.jar]
ivy_settings: %(pants_supportdir)s/ivy/fsqio/fsqio.publish.ivysettings.xml
# This is a dummy repo and is unused by Foursquare.
repos = {
    'oss_sonatype_repo': {
      'resolver': 'oss.sonatype.org-publish',
      'confs': ['default', 'sources'],
      'auth': 'build-support:oss.sonatype.org'
    }
  }
restrict_push_branches: master
restrict_push_urls: git@github.com:foursquare/fsqio.git


[repl.py]
# NOTE(mateo): Please keep the ipython version in sync with 3rdparty/python/requirements.txt.
ipython_requirements: ['ipython==5.0.0']
ipython: True


[mypy]
config_file: %(buildroot)s/build-support/fsqio/mypy/mypy.ini


[buildgen]
source_dirs: [
    'src',
    '3rdparty',
  ]
test_dirs: [
    'test',
  ]

target_whitelist: [
  'java_tests',
  'jsonc_config',
  'jsonc_library',
  'junit_tests',
  'python_library',
  'python_tests',
  'scala_library',
  'scala_record_library',
  'scala_soy_library',
  'scalac_plugin',
  'spindle_thrift_library'
  ]

managed_dependency_aliases: [
  'jar_library',
  'jsonc_config',
  'jsonc_library',
  'python_library',
  'python_requirement_library',
  'scala_library',
  'scala_soy_library',
  'scala_record_library',
  ]

buildgen_target_bags: [
    'src/thrift/io/fsq:all_thrift',
  ]


[buildgen.python]
fatal: True

first_party_packages = [
    'fsqio',
    'fsqio_test,'
  ]

ignored_prefixes: [
    'argparse',    # builtin, excluded because the 3rdparty version is installed transitively.
    'geojson',     # twofishes script in src/jvm.
    'lvm',         # linuxonly
    'shapely',     # twofishes script - requires c headers and not in virtualenv
  ]

# Surprised to see that py-zipkin is required here.
third_party_map: {
    'gen': {
      'apache': {
        'aurora': '3rdparty/python:apache.aurora.thrift',
        'thermos': '3rdparty/python:apache.aurora.thermos',
        },
      },
    'github3': {
      'DEFAULT': '3rdparty/python:github3.py',
    },
    'pants': {
      'DEFAULT': '3rdparty/python:pantsbuild.pants',
      'contrib': {
        'node': '3rdparty/python:pantsbuild.pants.contrib.node',
        'go': '3rdparty/python:pantsbuild.pants.contrib.go',
      },
    },
    'pants_test': {
      'DEFAULT': '3rdparty/python:pantsbuild.pants.testinfra',
    },
    'py_zipkin': {
      'DEFAULT': '3rdparty/python:py-zipkin',
    },
    'twitter': {
      'common': {
        'thrift': '3rdparty/python:twitter.common.service-thrift',
        },
      },
  }


[tag]
# Currently overridden internally, possible bug in dict_option.
by_prefix:  +{
    '3rdparty': ['exempt'],
    'test': ['tests'],
    'verification': ['tests'],
    'src': ['dependencies_cannot_have:tests'],

    'build-support/fsqio':  ['opensource', 'dependencies_must_have:opensource'],
    'src/jvm/io/fsq': ['opensource', 'dependencies_must_have:opensource'],
    'src/resources/io/fsq': ['opensource', 'dependencies_must_have:opensource'],
    'src/python/fsqio':  ['opensource', 'dependencies_must_have:opensource'],
    'src/thrift/io/fsq': ['opensource', 'dependencies_must_have:opensource'],
    'test/jvm/io/fsq': ['opensource', 'dependencies_must_have:opensource'],
    'test/python/fsqio':  ['opensource', 'dependencies_must_have:opensource'],
    'test/thrift/io/fsq': ['opensource', 'dependencies_must_have:opensource'],

    # Tests can rely on the spindle_codegen, but src dependencies are banned.
    'test': ['spindle_codegen'],

    'src/jvm/io/fsq/fhttp': ['fscommon'],
    'src/jvm/io/fsq/spindle/rogue': ['fscommon'],
    'src/jvm/io/fsq/spindle/runtime': ['fscommon'],
    'src/jvm/io/fsq/spindle/common/thrift': ['fscommon'],
    'src/jvm/io/fsq/rogue': ['fscommon'],
    'src/thrift/io/fsq/common': ['fscommon'],

    # NOTE(mateo): Another isolation case, where this is a proxy for 'dependencies_must_have_one_of: rogue, fscommon'.
    'src/jvm/io/fsq/common': ['fscommon', 'rogue'],
    'src/jvm/io/fsq/field': ['fscommon', 'rogue'],
  }


[resolve.ivy]
disable_reports: True
resolution_report_outdir: 3rdparty/reports/jvm
global_excludes: [
    {"org": "org.slf4j", "name": "slf4j-jdk14"},
    {"org": "org.slf4j", "name": "slf4j-log4j12"},
    {"org": "org.slf4j", "name": "slf4j-simple"},
    {"org": "log4j", "name": "log4j"},
    {"org": "log4j", "name": "apache-log4j-extras"},
    {"org": "org.apache.logging.log4j", "name": "log4j-core"},
    {"org": "org.apache.logging.log4j", "name": "log4j-api"},
    {"org": "org.mongodb", "name": "mongo-java-driver"},
    {"org": "com.esotericsoftware.kryo", "name": "kryo"},
    {"org": "com.esotericsoftware", "name": "reflectasm"},
    {"org": "org.apache.hadoop", "name": "hadoop-client"},
    {"org": "org.apache.hadoop", "name": "hadoop-core"},
    {"org": "io.netty", "name": "netty-buffer"},
    {"org": "io.netty", "name": "netty-codec"},
    {"org": "io.netty", "name": "netty-codec-dns"},
    {"org": "io.netty", "name": "netty-codec-haproxy"},
    {"org": "io.netty", "name": "netty-codec-http"},
    {"org": "io.netty", "name": "netty-codec-http2"},
    {"org": "io.netty", "name": "netty-codec-memcache"},
    {"org": "io.netty", "name": "netty-codec-mqtt"},
    {"org": "io.netty", "name": "netty-codec-smtp"},
    {"org": "io.netty", "name": "netty-codec-socks"},
    {"org": "io.netty", "name": "netty-codec-stomp"},
    {"org": "io.netty", "name": "netty-common"},
    {"org": "io.netty", "name": "netty-handler"},
    {"org": "io.netty", "name": "netty-handler-proxy"},
    {"org": "io.netty", "name": "netty-resolver"},
    {"org": "io.netty", "name": "netty-resolver-dns"},
    {"org": "io.netty", "name": "netty-transport"},
    {"org": "io.netty", "name": "netty-transport-rxtx"},
    {"org": "io.netty", "name": "netty-transport-sctp"},
    {"org": "io.netty", "name": "netty-transport-ut"},
    {"org": "org.jruby", "name": "jruby-complete"},
  ]


[cache.bundle]
# These are huge artifacts, but if bundle is upstream of your target, a few artifacts makes a difference.
max_entries_per_target = 1


[cache.binary]
# These are huge artifacts, but if bundle is upstream of your target, a few artifacts makes a difference.
max_entries_per_target = 1


[cache.test.junit]
ignore: False
read: True
write: True


[cache.test.pytest]
ignore: False
read: True
write: True


[cache.webpack.webpack-resolve]
# NOTE(mateo): This option is rolled into the task fingerprint - if it doesn't match the CI's config, the resulting
# artifacts will have mismatched cache_keys.
# Symlinks are not followed because node allows hanging symlinks as part of its fault-tolerant resolution process.
dereference_symlinks: False