Make the threshold for watchdog logging configurable

Summary:
## This stack

I am trying to debug a case of some methods stalling the entire reactor. See https://fb.workplace.com/groups/1708850869939124/permalink/1836334797190730/ for context.

## This diff

I've been trying to tune some of the "slower" call sites but there's a tone. Make the threshold configurable so we can start with the biggest ones first.

Reviewed By: clara-9

Differential Revision: D66166021

fbshipit-source-id: 6da925f21020acb3953baa58414e76af218da864

eden/mononoke/scs/scs_methods/src/source_control_impl.rs

@@ -117,6 +117,7 @@ pub struct SourceControlServiceImpl {
     identity_proxy_checker: Arc<ConnectionSecurityChecker>,
     pub(crate) acl_provider: Arc<dyn AclProvider>,
     pub(crate) enable_futures_watchdog: bool,
+    pub(crate) watchdog_max_poll: u64,
 }
 
 pub struct SourceControlServiceThriftImpl(Arc<SourceControlServiceImpl>);
@@ -136,6 +137,7 @@ impl SourceControlServiceImpl {
         factory_group: Option<Arc<FactoryGroup<2>>>,
         async_requests_queue: Option<Arc<AsyncMethodRequestQueue>>,
         enable_futures_watchdog: bool,
+        watchdog_max_poll: u64,
     ) -> Result<Self, anyhow::Error> {
         scuba_builder.add_common_server_data();
 
@@ -156,6 +158,7 @@ impl SourceControlServiceImpl {
             async_requests_queue,
             acl_provider: app.environment().acl_provider.clone(),
             enable_futures_watchdog,
+            watchdog_max_poll,
         })
     }
 
@@ -830,6 +833,7 @@ macro_rules! impl_thrift_methods {
                 let fut = async move {
                     let svc = self.0.clone();
                     let enable_futures_watchdog = self.0.enable_futures_watchdog;
+                    let watchdog_max_poll = self.0.watchdog_max_poll;
                     let (ctx, session_uuid) = create_ctx!(svc, $method_name, req_ctxt, $( $param_name ),*).await?;
                     let handler = {
                         cloned!(ctx);
@@ -843,7 +847,7 @@ macro_rules! impl_thrift_methods {
                                     f.watched(ctx.logger())
                                     .with_label(stringify!($method_name))
                                     .with_unique_id(&session_uuid)
-                                    .with_max_poll(50).await
+                                    .with_max_poll(watchdog_max_poll).await
                                 } else {
                                     f.await
                                 }

eden/mononoke/scs/scs_server/src/main.rs

@@ -124,6 +124,9 @@ struct ScsServerArgs {
     /// Enable the futures watchdog; this will log stack traces for futures that take longer than 0.5 seconds to complete.
     #[clap(long, default_value = "false")]
     enable_futures_watchdog: bool,
+    /// Sets the threshold for watchdog logging of top-level SCS methods. As a rule of thumb this should the same or lower than thrift_queue_timeout.
+    #[clap(long, default_value = "500")]
+    watchdog_method_max_poll: u64,
 }
 
 #[derive(ValueEnum, Clone, Copy, Debug, PartialEq, Eq)]
@@ -302,6 +305,7 @@ fn main(fb: FacebookInit) -> Result<(), Error> {
             maybe_factory_group,
             async_requests_queue_client,
             args.enable_futures_watchdog,
+            args.watchdog_method_max_poll,
         ))?
     };