From 4df420921441091c39cc8329d92eb8521cbc4c6b Mon Sep 17 00:00:00 2001 From: Deepak Dixit Date: Thu, 2 Feb 2023 03:04:23 +0530 Subject: [PATCH 01/26] Upgraded freemarker version to 2.3.32, also updated ftl version in FtlTemplateRenderer and MNode class (#565) https://freemarker.apache.org/docs/versions_2_3_32.html --- framework/build.gradle | 3 ++- .../org/moqui/impl/context/renderer/FtlTemplateRenderer.java | 2 +- framework/src/main/java/org/moqui/util/MNode.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/framework/build.gradle b/framework/build.gradle index 489ba9ebe..7c54bb5fa 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -115,7 +115,8 @@ dependencies { api 'com.vladsch.flexmark:flexmark-ext-toc:0.64.0' // Freemarker - api 'org.freemarker:freemarker:2.3.31' // Apache 2.0 + // Remember to change the version number in FtlTemplateRenderer and MNode class when upgrading + api 'org.freemarker:freemarker:2.3.32' // Apache 2.0 // H2 Database api 'com.h2database:h2:2.1.214' // MPL 2.0, EPL 1.0 diff --git a/framework/src/main/groovy/org/moqui/impl/context/renderer/FtlTemplateRenderer.java b/framework/src/main/groovy/org/moqui/impl/context/renderer/FtlTemplateRenderer.java index 2f4775163..f16e1cf2f 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/renderer/FtlTemplateRenderer.java +++ b/framework/src/main/groovy/org/moqui/impl/context/renderer/FtlTemplateRenderer.java @@ -38,7 +38,7 @@ @CompileStatic public class FtlTemplateRenderer implements TemplateRenderer { - public static final Version FTL_VERSION = Configuration.VERSION_2_3_30; + public static final Version FTL_VERSION = Configuration.VERSION_2_3_32; private static final Logger logger = LoggerFactory.getLogger(FtlTemplateRenderer.class); protected ExecutionContextFactoryImpl ecfi; diff --git a/framework/src/main/java/org/moqui/util/MNode.java b/framework/src/main/java/org/moqui/util/MNode.java index a60003f36..141a05312 100644 --- a/framework/src/main/java/org/moqui/util/MNode.java +++ b/framework/src/main/java/org/moqui/util/MNode.java @@ -42,7 +42,7 @@ @SuppressWarnings("unused") public class MNode implements TemplateNodeModel, TemplateSequenceModel, TemplateHashModelEx, AdapterTemplateModel, TemplateScalarModel { protected final static Logger logger = LoggerFactory.getLogger(MNode.class); - private static final Version FTL_VERSION = Configuration.VERSION_2_3_30; + private static final Version FTL_VERSION = Configuration.VERSION_2_3_32; private final static Map parsedNodeCache = new HashMap<>(); public static void clearParsedNodeCache() { parsedNodeCache.clear(); } From 5e01dce3da4875f93e2ea74b72c8a56b52fb0803 Mon Sep 17 00:00:00 2001 From: David E Jones Date: Wed, 1 Feb 2023 14:38:33 -0800 Subject: [PATCH 02/26] Various library updates: jackson-databind, jetty, joda-time, shiro, sl4jf, junit --- framework/build.gradle | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/framework/build.gradle b/framework/build.gradle index 7c54bb5fa..8bc029de6 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -27,7 +27,7 @@ buildscript { maven { url "https://plugins.gradle.org/m2/" } } dependencies { - classpath 'com.github.ben-manes:gradle-versions-plugin:0.42.0' + classpath 'com.github.ben-manes:gradle-versions-plugin:0.45.0' // uncomment to add the Error Prone compiler: classpath 'net.ltgt.gradle:gradle-errorprone-plugin:0.0.8' } } @@ -144,28 +144,28 @@ dependencies { api 'com.beust:jcommander:1.82' // Jackson Databind (JSON, etc) - api 'com.fasterxml.jackson.core:jackson-databind:2.13.4.2' + api 'com.fasterxml.jackson.core:jackson-databind:2.14.2' // Jetty HTTP Client and Proxy Servlet - api 'org.eclipse.jetty:jetty-client:10.0.12' // Apache 2.0 - api 'org.eclipse.jetty:jetty-proxy:10.0.12' // Apache 2.0 + api 'org.eclipse.jetty:jetty-client:10.0.13' // Apache 2.0 + api 'org.eclipse.jetty:jetty-proxy:10.0.13' // Apache 2.0 // javax.mail // NOTE: javax.mail depends on 'javax.activation:activation' which is the old package for 'javax.activation:javax.activation-api' used by jaxb-api api module('com.sun.mail:javax.mail:1.6.2') // CDDL // Joda Time (used by elasticsearch, aws) - api 'joda-time:joda-time:2.12.0' // Apache 2.0 + api 'joda-time:joda-time:2.12.2' // Apache 2.0 // JSoup (HTML parser, cleaner) api 'org.jsoup:jsoup:1.15.3' // MIT // Apache Shiro - api module('org.apache.shiro:shiro-core:1.10.0') // Apache 2.0 - api module('org.apache.shiro:shiro-web:1.10.0') // Apache 2.0 + api module('org.apache.shiro:shiro-core:1.11.0') // Apache 2.0 + api module('org.apache.shiro:shiro-web:1.11.0') // Apache 2.0 // SLF4J, Log4j 2 (note Log4j 2 is used by various libraries, best not to replace it even if mostly possible with SLF4J) - api 'org.slf4j:slf4j-api:2.0.3' + api 'org.slf4j:slf4j-api:2.0.6' implementation 'org.apache.logging.log4j:log4j-core:2.19.0' implementation 'org.apache.logging.log4j:log4j-api:2.19.0' runtimeOnly 'org.apache.logging.log4j:log4j-jcl:2.19.0' @@ -190,11 +190,11 @@ dependencies { // ========== test dependencies ========== // junit-platform-launcher is a dependency from spock-core, included explicitly to get more recent version as needed - testImplementation 'org.junit.platform:junit-platform-launcher:1.9.1' + testImplementation 'org.junit.platform:junit-platform-launcher:1.9.2' // junit-platform-suite required for test suites to specify test class order, etc - testImplementation 'org.junit.platform:junit-platform-suite:1.9.1' + testImplementation 'org.junit.platform:junit-platform-suite:1.9.2' // junit-jupiter-api for using JUnit directly, not generally needed for Spock based tests - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.1' + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.2' // Spock Framework testImplementation platform("org.spockframework:spock-bom:2.1-groovy-3.0") // Apache 2.0 testImplementation 'org.spockframework:spock-core:2.1-groovy-3.0' // Apache 2.0 @@ -203,16 +203,16 @@ dependencies { // ========== executable war dependencies ========== // Jetty - execWarRuntimeOnly 'org.eclipse.jetty:jetty-server:10.0.12' // Apache 2.0 - execWarRuntimeOnly 'org.eclipse.jetty:jetty-webapp:10.0.12' // Apache 2.0 - execWarRuntimeOnly 'org.eclipse.jetty:jetty-jndi:10.0.12' // Apache 2.0 - execWarRuntimeOnly 'org.eclipse.jetty.websocket:websocket-javax-server:10.0.12' // Apache 2.0 - execWarRuntimeOnly ('org.eclipse.jetty.websocket:websocket-javax-client:10.0.12') { // Apache 2.0 + execWarRuntimeOnly 'org.eclipse.jetty:jetty-server:10.0.13' // Apache 2.0 + execWarRuntimeOnly 'org.eclipse.jetty:jetty-webapp:10.0.13' // Apache 2.0 + execWarRuntimeOnly 'org.eclipse.jetty:jetty-jndi:10.0.13' // Apache 2.0 + execWarRuntimeOnly 'org.eclipse.jetty.websocket:websocket-javax-server:10.0.13' // Apache 2.0 + execWarRuntimeOnly ('org.eclipse.jetty.websocket:websocket-javax-client:10.0.13') { // Apache 2.0 exclude group: 'javax.websocket' } // we have the full websocket API, including the client one causes problems execWarRuntimeOnly 'javax.websocket:javax.websocket-api:1.1' - execWarRuntimeOnly ('org.eclipse.jetty.websocket:websocket-jetty-server:10.0.12') // Apache 2.0 + execWarRuntimeOnly ('org.eclipse.jetty.websocket:websocket-jetty-server:10.0.13') // Apache 2.0 // only include this if using Endpoint and MessageHandler annotations: - // execWarRuntime ('org.eclipse.jetty:jetty-annotations:10.0.12') // Apache 2.0 + // execWarRuntime ('org.eclipse.jetty:jetty-annotations:10.0.13') // Apache 2.0 execWarRuntimeOnly 'org.apache.logging.log4j:log4j-slf4j18-impl:2.18.0' } From 47828f0ba1ca3c156bf87dd69c509372fcd4ddc4 Mon Sep 17 00:00:00 2001 From: Deepak Dixit Date: Thu, 2 Feb 2023 11:21:41 +0530 Subject: [PATCH 03/26] Added missing package name in entity relationship --- framework/entity/ResourceEntities.xml | 2 +- framework/entity/ServerEntities.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/framework/entity/ResourceEntities.xml b/framework/entity/ResourceEntities.xml index 349e5139a..f9ac9b8cc 100644 --- a/framework/entity/ResourceEntities.xml +++ b/framework/entity/ResourceEntities.xml @@ -145,7 +145,7 @@ along with this software (see the LICENSE.md file). If not, see - + diff --git a/framework/entity/ServerEntities.xml b/framework/entity/ServerEntities.xml index f6f0481c9..17ec6556d 100644 --- a/framework/entity/ServerEntities.xml +++ b/framework/entity/ServerEntities.xml @@ -116,9 +116,9 @@ along with this software (see the LICENSE.md file). If not, see - + - + From 087749d131661e0c664190fc5b65da9ce570da6e Mon Sep 17 00:00:00 2001 From: David E Jones Date: Fri, 3 Feb 2023 15:35:29 -0800 Subject: [PATCH 04/26] Add StatusFlowTransitionFromAndTo view-entity --- framework/entity/BasicEntities.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/framework/entity/BasicEntities.xml b/framework/entity/BasicEntities.xml index 25615b2da..3ca7a10e4 100644 --- a/framework/entity/BasicEntities.xml +++ b/framework/entity/BasicEntities.xml @@ -373,6 +373,16 @@ along with this software (see the LICENSE.md file). If not, see + + + + + + + + + + From fbb65c214832c979c3558d0ec2df3d56aaccce60 Mon Sep 17 00:00:00 2001 From: David E Jones Date: Sat, 4 Feb 2023 04:46:22 -0800 Subject: [PATCH 05/26] In ServiceFacadeImpl add classes for Service LoadRunner, used for load testing of service calls with some profiling info, some basics for a first pass --- .../moqui/impl/context/ContextJavaUtil.java | 2 +- .../elastic/ElasticEntityListIterator.java | 1 + .../impl/service/ServiceCallAsyncImpl.groovy | 38 +++- .../impl/service/ServiceFacadeImpl.groovy | 199 ++++++++++++++++++ 4 files changed, 230 insertions(+), 10 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/ContextJavaUtil.java b/framework/src/main/groovy/org/moqui/impl/context/ContextJavaUtil.java index ea72322e4..7c9b62c90 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ContextJavaUtil.java +++ b/framework/src/main/groovy/org/moqui/impl/context/ContextJavaUtil.java @@ -735,7 +735,7 @@ static class CustomScheduledTask implements RunnableScheduledFuture { return "CustomScheduledTask " + (runnable != null ? runnable.getClass().getName() : (callable != null ? callable.getClass().getName() : "[no Runnable or Callable!]")); } } - static class CustomScheduledExecutor extends ScheduledThreadPoolExecutor { + public static class CustomScheduledExecutor extends ScheduledThreadPoolExecutor { public CustomScheduledExecutor(int coreThreads) { super(coreThreads, new ScheduledThreadFactory()); } diff --git a/framework/src/main/groovy/org/moqui/impl/entity/elastic/ElasticEntityListIterator.java b/framework/src/main/groovy/org/moqui/impl/entity/elastic/ElasticEntityListIterator.java index bf2c594d0..947193eaf 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/elastic/ElasticEntityListIterator.java +++ b/framework/src/main/groovy/org/moqui/impl/entity/elastic/ElasticEntityListIterator.java @@ -125,6 +125,7 @@ boolean nextResult() { // logger.warn("nextResult end resultCount " + resultCount + " overallIndex " + overallIndex + " currentListStartIndex " + currentListStartIndex + " currentDocList.size() " + currentDocList.size()); return hasCurrentValue(); } + @SuppressWarnings("unchecked") void fetchNext() { if (this.closed) throw new IllegalStateException("EntityListIterator is closed, cannot fetch next results"); diff --git a/framework/src/main/groovy/org/moqui/impl/service/ServiceCallAsyncImpl.groovy b/framework/src/main/groovy/org/moqui/impl/service/ServiceCallAsyncImpl.groovy index edf7903bb..3d4ddceeb 100644 --- a/framework/src/main/groovy/org/moqui/impl/service/ServiceCallAsyncImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/service/ServiceCallAsyncImpl.groovy @@ -102,6 +102,12 @@ class ServiceCallAsyncImpl extends ServiceCallImpl implements ServiceCallAsync { this.serviceName = serviceName this.parameters = new HashMap<>(parameters) } + AsyncServiceInfo(ExecutionContextFactoryImpl ecfi, String username, String serviceName, Map parameters) { + ecfiLocal = ecfi + threadUsername = username + this.serviceName = serviceName + this.parameters = new HashMap<>(parameters) + } @Override void writeExternal(ObjectOutput out) throws IOException { @@ -123,6 +129,9 @@ class ServiceCallAsyncImpl extends ServiceCallImpl implements ServiceCallAsync { } Map runInternal() throws Exception { + return runInternal(null, false) + } + Map runInternal(Map parameters, boolean skipEcCheck) throws Exception { ExecutionContextImpl threadEci = (ExecutionContextImpl) null try { // check for active Transaction @@ -135,22 +144,33 @@ class ServiceCallAsyncImpl extends ServiceCallImpl implements ServiceCallAsync { } } // check for active ExecutionContext - ExecutionContextImpl activeEc = getEcfi().activeContext.get() - if (activeEc != null) { - logger.error("In ServiceCallAsync service ${serviceName} there is already an ExecutionContext for user ${activeEc.user.username} (from ${activeEc.forThreadId}:${activeEc.forThreadName}) in this thread ${Thread.currentThread().id}:${Thread.currentThread().name}, destroying") - try { - activeEc.destroy() - } catch (Throwable t) { - logger.error("Error destroying ExecutionContext already in place in ServiceCallAsync in thread ${Thread.currentThread().id}:${Thread.currentThread().name}", t) + if (!skipEcCheck) { + ExecutionContextImpl activeEc = getEcfi().activeContext.get() + if (activeEc != null) { + logger.error("In ServiceCallAsync service ${serviceName} there is already an ExecutionContext for user ${activeEc.user.username} (from ${activeEc.forThreadId}:${activeEc.forThreadName}) in this thread ${Thread.currentThread().id}:${Thread.currentThread().name}, destroying") + try { + activeEc.destroy() + } catch (Throwable t) { + logger.error("Error destroying ExecutionContext already in place in ServiceCallAsync in thread ${Thread.currentThread().id}:${Thread.currentThread().name}", t) + } } } threadEci = getEcfi().getEci() - if (threadUsername != null && threadUsername.length() > 0) + if (threadUsername != null && threadUsername.length() > 0) { threadEci.userFacade.internalLoginUser(threadUsername, false) + } else { + threadEci.userFacade.loginAnonymousIfNoUser() + } + + Map parmsToUse = this.parameters + if (parameters != null) { + parmsToUse = new HashMap<>(this.parameters) + parmsToUse.putAll(parameters) + } // NOTE: authz is disabled because authz is checked before queueing - Map result = threadEci.serviceFacade.sync().name(serviceName).parameters(parameters).disableAuthz().call() + Map result = threadEci.serviceFacade.sync().name(serviceName).parameters(parmsToUse).disableAuthz().call() return result } catch (Throwable t) { logger.error("Error in async service", t) diff --git a/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy index f28eca2cc..167cc2461 100644 --- a/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy @@ -15,6 +15,7 @@ package org.moqui.impl.service import groovy.transform.CompileStatic import org.moqui.impl.context.ContextJavaUtil +import org.moqui.impl.context.ContextJavaUtil.CustomScheduledExecutor import org.moqui.resource.ResourceReference import org.moqui.context.ToolFactory import org.moqui.impl.context.ExecutionContextFactoryImpl @@ -33,6 +34,7 @@ import org.slf4j.LoggerFactory import javax.cache.Cache import javax.mail.internet.MimeMessage import java.util.concurrent.* +import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.locks.ReentrantLock @CompileStatic @@ -52,6 +54,7 @@ class ServiceFacadeImpl implements ServiceFacade { private ScheduledJobRunner jobRunner = null public final ThreadPoolExecutor jobWorkerPool + private LoadRunner loadRunner = null /** Distributed ExecutorService for async services, etc */ protected ExecutorService distributedExecutorService = null @@ -599,4 +602,200 @@ class ServiceFacadeImpl implements ServiceFacade { if (callbackList != null && callbackList.size() > 0) for (ServiceCallback scb in callbackList) scb.receiveEvent(context, t) } + + // ========================== + // Service LoadRunner Classes + // ========================== + + synchronized LoadRunner getLoadRunner() { + if (loadRunner == null) loadRunner = new LoadRunner(ecfi) + return loadRunner + } + + static class LoadRunnerServiceRunnable extends ServiceCallAsyncImpl.AsyncServiceInfo implements Runnable, Externalizable { + LoadRunner loadRunner + LoadRunnerServiceInfo serviceInfo + LoadRunnerServiceRunnable(String username, String serviceName, Map parameters, + LoadRunner loadRunner, LoadRunnerServiceInfo serviceInfo) { + super(loadRunner.ecfi, username, serviceName, parameters) + this.loadRunner = loadRunner + this.serviceInfo = serviceInfo + } + @Override void run() { + // TODO other parameters, maybe configurable with expanded strings? + String parametersExpr = serviceInfo.parametersExpr + Map parameters = [execIndex:loadRunner.execIndex.getAndIncrement()] as Map + if (parametersExpr != null && !parametersExpr.isEmpty()) { + try { + Map exprMap = (Map) loadRunner.ecfi.getEci().resourceFacade + .expression(parametersExpr, null, parameters) + if (exprMap != null) parameters.putAll(exprMap) + } catch (Throwable t) { + logger.error("Error in Service LoadRunner parameter expression: ${parametersExpr}", t) + } + } + + long startTime = System.currentTimeMillis() + try { + serviceInfo.lastResult = runInternal(parameters, true) + } catch (Throwable t) { + // logged elsewhere, just swallow and count + serviceInfo.errorCount++ + } + long endTime = System.currentTimeMillis() + long runTime = endTime - startTime + serviceInfo.runCount++ + serviceInfo.lastRunTime = endTime + serviceInfo.totalTime += runTime + serviceInfo.totalSquaredTime += runTime * runTime + } + } + static class LoadRunnerServiceInfo { + String serviceName, parametersExpr = null + int targetThreads, runDelayMs, rampDelayMs + AtomicInteger currentThreads = new AtomicInteger(0) + long runCount = 0, errorCount = 0, lastRunTime = 0, beginTime = 0, totalTime = 0, totalSquaredTime = 0 + Map lastResult = null + LoadRunnerServiceInfo(String serviceName, int targetThreads, int runDelayMs, int rampDelayMs) { + this.serviceName = serviceName + this.targetThreads = targetThreads + this.runDelayMs = runDelayMs + this.rampDelayMs = rampDelayMs + } + void addThread(LoadRunner loadRunner) { + LoadRunnerServiceRunnable runnable = + new LoadRunnerServiceRunnable(null, serviceName, [:], loadRunner, this) + // NOTE: use scheduleWithFixedDelay so delay is wait between terminate of one and start of another, better for both short and long running test runs + loadRunner.scheduledExecutor.scheduleWithFixedDelay(runnable, 1, runDelayMs, TimeUnit.MILLISECONDS) + } + void addRampThread(LoadRunner loadRunner) { + beginTime = System.currentTimeMillis() + LoadRunnerRamperRunnable runnable = new LoadRunnerRamperRunnable(loadRunner, this) + // NOTE: use scheduleAtFixedRate so one is added each delay period regardless of how long it takes (generally not long) + loadRunner.scheduledExecutor.scheduleAtFixedRate(runnable, 1, rampDelayMs, TimeUnit.MILLISECONDS) + } + } + static class LoadRunnerRamperRunnable implements Runnable { + LoadRunner loadRunner + LoadRunnerServiceInfo serviceInfo + LoadRunnerRamperRunnable(LoadRunner loadRunner, LoadRunnerServiceInfo serviceInfo) { + this.loadRunner = loadRunner + this.serviceInfo = serviceInfo + } + @Override void run() { + if (serviceInfo.currentThreads < serviceInfo.targetThreads) { + serviceInfo.addThread(loadRunner) + // may not actually need AtomicInteger here, but for ramp down will need a list of ScheduledFuture objects and might be useful there + serviceInfo.currentThreads.incrementAndGet() + } + // TODO add delayed ramp-down, useful for some performance behavior patterns but usually redundant with delayed ramp up to look for elbows in the response time over time + } + } + static class LoadRunner { + ExecutionContextFactoryImpl ecfi + CustomScheduledExecutor scheduledExecutor = null + ArrayList serviceInfos = new ArrayList<>() + int corePoolSize = 4, maxPoolSize = 8 + AtomicInteger execIndex = new AtomicInteger(1) + ReentrantLock mutateLock = new ReentrantLock() + + LoadRunner(ExecutionContextFactoryImpl ecfi) { + this.ecfi = ecfi + } + + void setServiceInfo(String serviceName, String parametersExpr, int targetThreads, int runDelayMs, int rampDelayMs) { + mutateLock.lock() + try { + LoadRunnerServiceInfo serviceInfo = null + for (int i = 0; i < serviceInfos.size(); i++) { + LoadRunnerServiceInfo curInfo = (LoadRunnerServiceInfo) serviceInfos.get(i) + if (curInfo.serviceName == serviceName) serviceInfo = curInfo + } + if (serviceInfo == null) { + serviceInfo = new LoadRunnerServiceInfo(serviceName, targetThreads, runDelayMs, rampDelayMs) + serviceInfo.parametersExpr = parametersExpr + + serviceInfos.add(serviceInfo) + + if (scheduledExecutor != null) { + // begin() already called, get this started + serviceInfo.addRampThread(this) + } + } else { + serviceInfo.parametersExpr = parametersExpr + serviceInfo.targetThreads = targetThreads + serviceInfo.runDelayMs = runDelayMs + serviceInfo.rampDelayMs = rampDelayMs + } + } finally { + mutateLock.unlock() + } + } + void begin() { + mutateLock.lock() + try { + // TODO set maxPoolSize to CPU count x2 or something if needed to scale the load runner itself; probably not much as services running on same system... + if (scheduledExecutor == null) { + // restart index + execIndex = new AtomicInteger(1) + + for (int i = 0; i < serviceInfos.size(); i++) { + LoadRunnerServiceInfo curInfo = (LoadRunnerServiceInfo) serviceInfos.get(i) + // clear out stats before start + curInfo.currentThreads = new AtomicInteger(0) + curInfo.runCount = 0 + curInfo.errorCount = 0 + curInfo.lastRunTime = 0 + curInfo.beginTime = 0 + curInfo.totalTime = 0 + curInfo.totalSquaredTime = 0 + curInfo.lastResult = null + } + + scheduledExecutor = new CustomScheduledExecutor(corePoolSize) + scheduledExecutor.setMaximumPoolSize(maxPoolSize) + + for (int i = 0; i < serviceInfos.size(); i++) { + LoadRunnerServiceInfo curInfo = (LoadRunnerServiceInfo) serviceInfos.get(i) + // do this once here + curInfo.addThread(this) + // add a schedule for the rest + curInfo.addRampThread(this) + } + } + } finally { + mutateLock.unlock() + } + } + void stopNow() { + mutateLock.lock() + try { + if (scheduledExecutor != null) { + logger.info("Shutting down LoadRunner ScheduledExecutorService now") + scheduledExecutor.shutdownNow() + scheduledExecutor = null + } + } finally { + mutateLock.unlock() + } + } + void stopWait() { + mutateLock.lock() + try { + if (scheduledExecutor != null) { + logger.info("Shutting down LoadRunner ScheduledExecutorService") + scheduledExecutor.shutdown() + } + + if (scheduledExecutor != null) { + scheduledExecutor.awaitTermination(30, TimeUnit.SECONDS) + if (scheduledExecutor.isTerminated()) logger.info("LoadRunner Scheduled executor shut down and terminated") + else logger.warn("LoadRunner Scheduled executor NOT YET terminated, waited 30 seconds") + scheduledExecutor = null + } + } finally { + mutateLock.unlock() + } + } + } } From 4558bdf0b901dd7df156ae1bd5eaf206b3e33c37 Mon Sep 17 00:00:00 2001 From: David E Jones Date: Sat, 4 Feb 2023 13:30:19 -0800 Subject: [PATCH 06/26] In Service LoadRunner add stats broken down by artifact type with detail, part of code is more generic addition to ArtifactExecutionInfoImpl --- .../ArtifactExecutionFacadeImpl.groovy | 4 + .../context/ArtifactExecutionInfoImpl.java | 118 ++++++++++++++++++ .../impl/service/ServiceFacadeImpl.groovy | 81 +++++++++--- .../java/org/moqui/util/ObjectUtilities.java | 46 +++++++ 4 files changed, 229 insertions(+), 20 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionFacadeImpl.groovy index 82e9acb3d..5edf7ab96 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionFacadeImpl.groovy @@ -172,6 +172,10 @@ class ArtifactExecutionFacadeImpl implements ArtifactExecutionFacade { return sw.toString() } + ArtifactExecutionInfoImpl.ArtifactTypeStats getArtifactTypeStats() { + return ArtifactExecutionInfoImpl.getArtifactTypeStats(artifactExecutionInfoHistory) + } + void logProfilingDetail() { if (!logger.isInfoEnabled()) return diff --git a/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionInfoImpl.java b/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionInfoImpl.java index 5880cf390..084d2e04b 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionInfoImpl.java +++ b/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionInfoImpl.java @@ -17,6 +17,8 @@ import org.moqui.impl.entity.EntityValueBase; import org.moqui.util.CollectionUtilities; import org.moqui.util.StringUtilities; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.StringWriter; @@ -26,6 +28,7 @@ import java.util.*; public class ArtifactExecutionInfoImpl implements ArtifactExecutionInfo { + protected final static Logger logger = LoggerFactory.getLogger(ArtifactExecutionInfoImpl.class); // NOTE: these need to be in a Map instead of the DB because Enumeration records may not yet be loaded private final static Map artifactTypeDescriptionMap = new EnumMap<>(ArtifactType.class); @@ -153,6 +156,7 @@ public void copyAuthorizedInfo(ArtifactExecutionInfoImpl aeii) { @Override public long getRunningTime() { return endTimeNanos != 0 ? endTimeNanos - startTimeNanos : 0; } public double getRunningTimeMillisDouble() { return (endTimeNanos != 0 ? endTimeNanos - startTimeNanos : 0) / 1000000.0; } + public long getRunningTimeMillisLong() { return Math.round((endTimeNanos != 0 ? endTimeNanos - startTimeNanos : 0) / 1000000.0); } private void calcChildTime(boolean recurse) { childrenRunningTime = 0; if (childList != null) for (ArtifactExecutionInfoImpl aeii: childList) { @@ -211,6 +215,120 @@ public void print(Writer writer, int level, boolean children) { } private String getKeyString() { return nameInternal + ":" + internalTypeEnum.name() + ":" + internalActionEnum.name() + ":" + actionDetail; } + private String getKeyStringNoName() { return internalTypeEnum.name() + ":" + internalActionEnum.name() + ":" + actionDetail; } + + public static class ArtifactTypeStats { + public int screenCount = 0, screenTransCount = 0, screenContentCount = 0, restPathCount = 0, + serviceViewCount = 0, serviceOtherCount = 0, + entityFindOneCount = 0, entityFindListCount = 0, entityFindIteratorCount = 0, entityFindCountCount = 0, + entityCreateCount = 0, entityUpdateCount = 0, entityDeleteCount = 0; + public long screenTime = 0, screenTransTime = 0, screenContentTime = 0, restPathTime = 0, + serviceViewTime = 0, serviceOtherTime = 0, + entityFindOneTime = 0, entityFindListTime = 0, entityFindIteratorTime = 0, entityFindCountTime = 0, + entityCreateTime = 0, entityUpdateTime = 0, entityDeleteTime = 0; + public void add(ArtifactTypeStats that) { + screenCount += that.screenCount; screenTransCount += that.screenTransCount; + screenContentCount += that.screenContentCount; restPathCount += that.restPathCount; + serviceViewCount += that.serviceViewCount; serviceOtherCount += that.serviceOtherCount; + entityFindOneCount += that.entityFindOneCount; entityFindListCount += that.entityFindListCount; + entityFindIteratorCount += that.entityFindIteratorCount; entityFindCountCount += that.entityFindCountCount; + entityCreateCount += that.entityCreateCount; entityUpdateCount += that.entityUpdateCount; + entityDeleteCount += that.entityDeleteCount; + + screenTime += that.screenTime; screenTransTime += that.screenTransTime; + screenContentTime += that.screenContentTime; restPathTime += that.restPathTime; + serviceViewTime += that.serviceViewTime; serviceOtherTime += that.serviceOtherTime; + entityFindOneTime += that.entityFindOneTime; entityFindListTime += that.entityFindListTime; + entityFindIteratorTime += that.entityFindIteratorTime; entityFindCountTime += that.entityFindCountTime; + entityCreateTime += that.entityCreateTime; entityUpdateTime += that.entityUpdateTime; + entityDeleteTime += that.entityDeleteTime; + } + } + static ArtifactTypeStats getArtifactTypeStats(ArrayList aeiiList) { + ArtifactTypeStats stats = new ArtifactTypeStats(); + addArtifactTypeStats(aeiiList, stats); + return stats; + } + static void addArtifactTypeStats(ArrayList aeiiList, ArtifactTypeStats stats) { + if (aeiiList == null) return; + int aeiiListSize = aeiiList.size(); + for (int i = 0; i < aeiiListSize; i++) { + ArtifactExecutionInfoImpl aeii = aeiiList.get(i); + // tight loop, use switch instead of if on these enums for much better performance; run fast for use in on the fly accumulators + switch (aeii.internalTypeEnum) { + case AT_ENTITY: + switch (aeii.internalActionEnum) { + case AUTHZA_VIEW: + if (aeii.actionDetail != null && !aeii.actionDetail.isEmpty()) { + char first = aeii.actionDetail.charAt(0); + switch (first) { + case 'o': // one + case 'r': // refresh + stats.entityFindOneCount++; + stats.entityFindOneTime += aeii.getRunningTime(); + break; + case 'l': // list + stats.entityFindListCount++; + stats.entityFindListTime += aeii.getRunningTime(); + break; + case 'i': // iterator + stats.entityFindIteratorCount++; + stats.entityFindIteratorTime += aeii.getRunningTime(); + break; + case 'c': // count + stats.entityFindCountCount++; + stats.entityFindCountTime += aeii.getRunningTime(); + break; + } + } else { + logger.warn("entity view with no detail " + aeii.toBasicString()); + } + break; + case AUTHZA_CREATE: + stats.entityCreateCount++; + stats.entityCreateTime += aeii.getRunningTime(); + break; + case AUTHZA_UPDATE: + stats.entityUpdateCount++; + stats.entityUpdateTime += aeii.getRunningTime(); + break; + case AUTHZA_DELETE: + stats.entityDeleteCount++; + stats.entityDeleteTime += aeii.getRunningTime(); + break; + } + break; + case AT_SERVICE: + if (aeii.internalActionEnum == AUTHZA_VIEW) { + stats.serviceViewCount++; + stats.serviceViewTime += aeii.getRunningTime(); + } else { + stats.serviceOtherCount++; + stats.serviceOtherTime += aeii.getRunningTime(); + } + break; + case AT_XML_SCREEN: + stats.screenCount++; + stats.screenTime += aeii.getRunningTime(); + break; + case AT_XML_SCREEN_TRANS: + stats.screenTransCount++; + stats.screenTransTime += aeii.getRunningTime(); + break; + case AT_XML_SCREEN_CONTENT: + stats.screenContentCount++; + stats.screenContentTime += aeii.getRunningTime(); + break; + case AT_REST_PATH: + stats.restPathCount++; + stats.restPathTime += aeii.getRunningTime(); + break; + } + + // this aeii is done, how about children? + addArtifactTypeStats(aeii.childList, stats); + } + } @SuppressWarnings("unchecked") static List> hotSpotByTime(List aeiiList, boolean ownTime, String orderBy) { diff --git a/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy index 167cc2461..c7b0087f3 100644 --- a/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy @@ -14,6 +14,8 @@ package org.moqui.impl.service import groovy.transform.CompileStatic +import org.moqui.impl.context.ArtifactExecutionInfoImpl +import org.moqui.impl.context.ArtifactExecutionInfoImpl.ArtifactTypeStats import org.moqui.impl.context.ContextJavaUtil import org.moqui.impl.context.ContextJavaUtil.CustomScheduledExecutor import org.moqui.resource.ResourceReference @@ -612,19 +614,41 @@ class ServiceFacadeImpl implements ServiceFacade { return loadRunner } - static class LoadRunnerServiceRunnable extends ServiceCallAsyncImpl.AsyncServiceInfo implements Runnable, Externalizable { + static class LoadRunnerServiceRunnable implements Runnable { + ExecutionContextFactoryImpl ecfi + String serviceName LoadRunner loadRunner LoadRunnerServiceInfo serviceInfo - LoadRunnerServiceRunnable(String username, String serviceName, Map parameters, - LoadRunner loadRunner, LoadRunnerServiceInfo serviceInfo) { - super(loadRunner.ecfi, username, serviceName, parameters) + + LoadRunnerServiceRunnable(String serviceName, LoadRunner loadRunner, LoadRunnerServiceInfo serviceInfo) { + this.ecfi = loadRunner.ecfi + this.serviceName = serviceName this.loadRunner = loadRunner this.serviceInfo = serviceInfo } @Override void run() { - // TODO other parameters, maybe configurable with expanded strings? + // check for active Transaction + if (getEcfi().transactionFacade.isTransactionInPlace()) { + logger.error("In LoadRunner service ${serviceName} a transaction is in place for thread ${Thread.currentThread().getName()}, trying to commit") + try { + getEcfi().transactionFacade.destroyAllInThread() + } catch (Exception e) { + logger.error("LoadRunner commit in place transaction failed for thread ${Thread.currentThread().getName()}", e) + } + } + // check for active ExecutionContext + ExecutionContextImpl activeEc = getEcfi().activeContext.get() + if (activeEc != null) { + logger.error("In LoadRunner service ${serviceName} there is already an ExecutionContext for user ${activeEc.user.username} (from ${activeEc.forThreadId}:${activeEc.forThreadName}) in this thread ${Thread.currentThread().id}:${Thread.currentThread().name}, destroying") + try { + activeEc.destroy() + } catch (Throwable t) { + logger.error("Error destroying LoadRunner already in place in ServiceCallAsync in thread ${Thread.currentThread().id}:${Thread.currentThread().name}", t) + } + } + String parametersExpr = serviceInfo.parametersExpr - Map parameters = [execIndex:loadRunner.execIndex.getAndIncrement()] as Map + Map parameters = [index:loadRunner.execIndex.getAndIncrement()] as Map if (parametersExpr != null && !parametersExpr.isEmpty()) { try { Map exprMap = (Map) loadRunner.ecfi.getEci().resourceFacade @@ -636,26 +660,44 @@ class ServiceFacadeImpl implements ServiceFacade { } long startTime = System.currentTimeMillis() + ExecutionContextImpl threadEci = ecfi.getEci() try { - serviceInfo.lastResult = runInternal(parameters, true) - } catch (Throwable t) { - // logged elsewhere, just swallow and count - serviceInfo.errorCount++ + // always login anonymous, disable authz below + threadEci.userFacade.loginAnonymousIfNoUser() + + // run the service + try { + serviceInfo.lastResult = threadEci.serviceFacade.sync().name(serviceName).parameters(parameters).disableAuthz().call() + } catch (Throwable t) { + // logged elsewhere, just count and swallow + serviceInfo.errorCount++ + } + + serviceInfo.artifactTypeStats.add(threadEci.artifactExecutionFacade.getArtifactTypeStats()) + } finally { + if (threadEci != null) threadEci.destroy() } + long endTime = System.currentTimeMillis() long runTime = endTime - startTime serviceInfo.runCount++ serviceInfo.lastRunTime = endTime serviceInfo.totalTime += runTime serviceInfo.totalSquaredTime += runTime * runTime + if (runTime < serviceInfo.minTime) serviceInfo.minTime = runTime + if (runTime > serviceInfo.maxTime) serviceInfo.maxTime = runTime } } static class LoadRunnerServiceInfo { String serviceName, parametersExpr = null int targetThreads, runDelayMs, rampDelayMs AtomicInteger currentThreads = new AtomicInteger(0) - long runCount = 0, errorCount = 0, lastRunTime = 0, beginTime = 0, totalTime = 0, totalSquaredTime = 0 + + long lastRunTime = 0, beginTime = 0, totalTime = 0, totalSquaredTime = 0, minTime = Long.MAX_VALUE, maxTime = 0 + int runCount = 0, errorCount = 0 + ArtifactTypeStats artifactTypeStats = new ArtifactTypeStats() Map lastResult = null + LoadRunnerServiceInfo(String serviceName, int targetThreads, int runDelayMs, int rampDelayMs) { this.serviceName = serviceName this.targetThreads = targetThreads @@ -663,8 +705,7 @@ class ServiceFacadeImpl implements ServiceFacade { this.rampDelayMs = rampDelayMs } void addThread(LoadRunner loadRunner) { - LoadRunnerServiceRunnable runnable = - new LoadRunnerServiceRunnable(null, serviceName, [:], loadRunner, this) + LoadRunnerServiceRunnable runnable = new LoadRunnerServiceRunnable(serviceName, loadRunner, this) // NOTE: use scheduleWithFixedDelay so delay is wait between terminate of one and start of another, better for both short and long running test runs loadRunner.scheduledExecutor.scheduleWithFixedDelay(runnable, 1, runDelayMs, TimeUnit.MILLISECONDS) } @@ -674,6 +715,12 @@ class ServiceFacadeImpl implements ServiceFacade { // NOTE: use scheduleAtFixedRate so one is added each delay period regardless of how long it takes (generally not long) loadRunner.scheduledExecutor.scheduleAtFixedRate(runnable, 1, rampDelayMs, TimeUnit.MILLISECONDS) } + void resetStats() { + lastRunTime = 0; beginTime = 0; totalTime = 0; totalSquaredTime = 0; minTime = Long.MAX_VALUE; maxTime = 0 + runCount = 0; errorCount = 0 + artifactTypeStats = new ArtifactTypeStats() + lastResult = null + } } static class LoadRunnerRamperRunnable implements Runnable { LoadRunner loadRunner @@ -743,13 +790,7 @@ class ServiceFacadeImpl implements ServiceFacade { LoadRunnerServiceInfo curInfo = (LoadRunnerServiceInfo) serviceInfos.get(i) // clear out stats before start curInfo.currentThreads = new AtomicInteger(0) - curInfo.runCount = 0 - curInfo.errorCount = 0 - curInfo.lastRunTime = 0 - curInfo.beginTime = 0 - curInfo.totalTime = 0 - curInfo.totalSquaredTime = 0 - curInfo.lastResult = null + curInfo.resetStats() } scheduledExecutor = new CustomScheduledExecutor(corePoolSize) diff --git a/framework/src/main/java/org/moqui/util/ObjectUtilities.java b/framework/src/main/java/org/moqui/util/ObjectUtilities.java index 1be291e92..f2b794603 100644 --- a/framework/src/main/java/org/moqui/util/ObjectUtilities.java +++ b/framework/src/main/java/org/moqui/util/ObjectUtilities.java @@ -18,7 +18,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.beans.BeanInfo; +import java.beans.IntrospectionException; +import java.beans.Introspector; +import java.beans.PropertyDescriptor; import java.io.*; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.math.BigDecimal; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -53,6 +60,45 @@ public class ObjectUtilities { temporalUnitByUomId = tum; } + /** Populate a Map with public fields and Java Bean style properties (using java.beans.BeanInfo) */ + public static Map objectToMap(Object bean) { + if (bean == null) return null; + Map map = new HashMap<>(); + + Class clazz = bean.getClass(); + Field[] fields = clazz.getFields(); + for (int fi = 0; fi < fields.length; fi++) { + Field field = fields[fi]; + try { + map.put(field.getName(), field.get(bean)); + } catch (IllegalAccessException e) { + // do nothing, maybe log at some point if we care enough and are okay with the potential performance hit + } + } + + try { + BeanInfo beanInfo = Introspector.getBeanInfo(clazz); + PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors(); + for (int pi = 0; pi < propertyDescriptors.length; pi++) { + PropertyDescriptor propertyDescriptor = propertyDescriptors[pi]; + Method readMethod = propertyDescriptor.getReadMethod(); + if (readMethod != null) { + try { + map.put(propertyDescriptor.getName(), readMethod.invoke(bean)); + } catch (IllegalAccessException | InvocationTargetException e) { + // nothing again + } + } + } + } catch (IntrospectionException e) { + // nothing again + } + + // this gets picked up automatically, just remove at the end, faster than checking along the way + map.remove("class"); + + return map; + } @SuppressWarnings("unchecked") public static Object basicConvert(Object value, final String javaType) { From 9ed14836059de5083d44b891d44dbef4e419103e Mon Sep 17 00:00:00 2001 From: David E Jones Date: Sat, 4 Feb 2023 19:40:39 -0800 Subject: [PATCH 07/26] Service LoadRunner improved stats gathering with time bin based stats in addition to entire run stats, prep work for performance charts and such --- .../context/ArtifactExecutionInfoImpl.java | 8 + .../moqui/impl/context/ContextJavaUtil.java | 9 +- .../impl/service/ServiceFacadeImpl.groovy | 198 ++++++++++++++---- .../java/org/moqui/util/ObjectUtilities.java | 2 + 4 files changed, 171 insertions(+), 46 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionInfoImpl.java b/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionInfoImpl.java index 084d2e04b..67fe40d54 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionInfoImpl.java +++ b/framework/src/main/groovy/org/moqui/impl/context/ArtifactExecutionInfoImpl.java @@ -218,6 +218,7 @@ public void print(Writer writer, int level, boolean children) { private String getKeyStringNoName() { return internalTypeEnum.name() + ":" + internalActionEnum.name() + ":" + actionDetail; } public static class ArtifactTypeStats { + public int screenCount = 0, screenTransCount = 0, screenContentCount = 0, restPathCount = 0, serviceViewCount = 0, serviceOtherCount = 0, entityFindOneCount = 0, entityFindListCount = 0, entityFindIteratorCount = 0, entityFindCountCount = 0, @@ -227,6 +228,8 @@ public static class ArtifactTypeStats { entityFindOneTime = 0, entityFindListTime = 0, entityFindIteratorTime = 0, entityFindCountTime = 0, entityCreateTime = 0, entityUpdateTime = 0, entityDeleteTime = 0; public void add(ArtifactTypeStats that) { + if (that == null) return; + screenCount += that.screenCount; screenTransCount += that.screenTransCount; screenContentCount += that.screenContentCount; restPathCount += that.restPathCount; serviceViewCount += that.serviceViewCount; serviceOtherCount += that.serviceOtherCount; @@ -243,6 +246,11 @@ public void add(ArtifactTypeStats that) { entityCreateTime += that.entityCreateTime; entityUpdateTime += that.entityUpdateTime; entityDeleteTime += that.entityDeleteTime; } + public ArtifactTypeStats cloneStats(ArtifactTypeStats that) { + ArtifactTypeStats newStats = new ArtifactTypeStats(); + newStats.add(that); + return newStats; + } } static ArtifactTypeStats getArtifactTypeStats(ArrayList aeiiList) { ArtifactTypeStats stats = new ArtifactTypeStats(); diff --git a/framework/src/main/groovy/org/moqui/impl/context/ContextJavaUtil.java b/framework/src/main/groovy/org/moqui/impl/context/ContextJavaUtil.java index 7c9b62c90..9f2321e38 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ContextJavaUtil.java +++ b/framework/src/main/groovy/org/moqui/impl/context/ContextJavaUtil.java @@ -695,17 +695,17 @@ static class ScheduledThreadFactory implements ThreadFactory { private final AtomicInteger threadNumber = new AtomicInteger(1); public Thread newThread(Runnable r) { return new Thread(workerGroup, r, "MoquiScheduled-" + threadNumber.getAndIncrement()); } } - static class CustomScheduledTask implements RunnableScheduledFuture { + public static class CustomScheduledTask implements RunnableScheduledFuture { public final Runnable runnable; public final Callable callable; public final RunnableScheduledFuture future; - CustomScheduledTask(Runnable runnable, RunnableScheduledFuture future) { + public CustomScheduledTask(Runnable runnable, RunnableScheduledFuture future) { this.runnable = runnable; this.callable = null; this.future = future; } - CustomScheduledTask(Callable callable, RunnableScheduledFuture future) { + public CustomScheduledTask(Callable callable, RunnableScheduledFuture future) { this.runnable = null; this.callable = callable; this.future = future; @@ -739,6 +739,9 @@ public static class CustomScheduledExecutor extends ScheduledThreadPoolExecutor public CustomScheduledExecutor(int coreThreads) { super(coreThreads, new ScheduledThreadFactory()); } + public CustomScheduledExecutor(int coreThreads, ThreadFactory threadFactory) { + super(coreThreads, threadFactory); + } protected RunnableScheduledFuture decorateTask(Runnable r, RunnableScheduledFuture task) { return new CustomScheduledTask(r, task); } diff --git a/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy index c7b0087f3..b6c5fb52f 100644 --- a/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy @@ -14,6 +14,7 @@ package org.moqui.impl.service import groovy.transform.CompileStatic +import org.moqui.Moqui import org.moqui.impl.context.ArtifactExecutionInfoImpl import org.moqui.impl.context.ArtifactExecutionInfoImpl.ArtifactTypeStats import org.moqui.impl.context.ContextJavaUtil @@ -28,6 +29,7 @@ import org.moqui.impl.service.runner.RemoteJsonRpcServiceRunner import org.moqui.service.* import org.moqui.util.CollectionUtilities import org.moqui.util.MNode +import org.moqui.util.ObjectUtilities import org.moqui.util.RestClient import org.moqui.util.StringUtilities import org.slf4j.Logger @@ -35,6 +37,7 @@ import org.slf4j.LoggerFactory import javax.cache.Cache import javax.mail.internet.MimeMessage +import java.sql.Timestamp import java.util.concurrent.* import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.locks.ReentrantLock @@ -614,19 +617,43 @@ class ServiceFacadeImpl implements ServiceFacade { return loadRunner } - static class LoadRunnerServiceRunnable implements Runnable { - ExecutionContextFactoryImpl ecfi - String serviceName - LoadRunner loadRunner - LoadRunnerServiceInfo serviceInfo + static class LoadRunnerServiceRunnable implements Runnable, Externalizable { + volatile ExecutionContextFactoryImpl ecfi + volatile LoadRunner loadRunner + String serviceName, parametersExpr - LoadRunnerServiceRunnable(String serviceName, LoadRunner loadRunner, LoadRunnerServiceInfo serviceInfo) { + LoadRunnerServiceRunnable() { + // init the other objects that can't be serialized + ecfi = (ExecutionContextFactoryImpl) Moqui.getExecutionContextFactory() + loadRunner = ecfi.serviceFacade.getLoadRunner() + } + LoadRunnerServiceRunnable(String serviceName, String parametersExpr, LoadRunner loadRunner) { + this.loadRunner = loadRunner this.ecfi = loadRunner.ecfi this.serviceName = serviceName - this.loadRunner = loadRunner - this.serviceInfo = serviceInfo + this.parametersExpr = parametersExpr + } + + @Override + void writeExternal(ObjectOutput out) throws IOException { + out.writeUTF(serviceName) // never null + out.writeObject(parametersExpr) // may be null } + + @Override + void readExternal(ObjectInput objectInput) throws IOException, ClassNotFoundException { + serviceName = objectInput.readUTF() + parametersExpr = objectInput.readObject() + } + @Override void run() { + try { + runInternal() + } catch (Throwable t) { + logger.error("Error in LoadRunner service run", t) + } + } + void runInternal() { // check for active Transaction if (getEcfi().transactionFacade.isTransactionInPlace()) { logger.error("In LoadRunner service ${serviceName} a transaction is in place for thread ${Thread.currentThread().getName()}, trying to commit") @@ -647,6 +674,11 @@ class ServiceFacadeImpl implements ServiceFacade { } } + LoadRunnerServiceInfo serviceInfo = loadRunner.getServiceInfo(serviceName, parametersExpr) + if (serviceInfo == null) { + logger.error("Service Info not found for ${serviceName} ${parametersExpr}, not running") + return + } String parametersExpr = serviceInfo.parametersExpr Map parameters = [index:loadRunner.execIndex.getAndIncrement()] as Map if (parametersExpr != null && !parametersExpr.isEmpty()) { @@ -673,53 +705,112 @@ class ServiceFacadeImpl implements ServiceFacade { serviceInfo.errorCount++ } - serviceInfo.artifactTypeStats.add(threadEci.artifactExecutionFacade.getArtifactTypeStats()) + // count the run and accumulate stats + serviceInfo.countRun(loadRunner, startTime, System.currentTimeMillis(), + threadEci.artifactExecutionFacade.getArtifactTypeStats()) } finally { if (threadEci != null) threadEci.destroy() } - - long endTime = System.currentTimeMillis() - long runTime = endTime - startTime - serviceInfo.runCount++ - serviceInfo.lastRunTime = endTime - serviceInfo.totalTime += runTime - serviceInfo.totalSquaredTime += runTime * runTime - if (runTime < serviceInfo.minTime) serviceInfo.minTime = runTime - if (runTime > serviceInfo.maxTime) serviceInfo.maxTime = runTime } } - static class LoadRunnerServiceInfo { - String serviceName, parametersExpr = null - int targetThreads, runDelayMs, rampDelayMs - AtomicInteger currentThreads = new AtomicInteger(0) - + static class LoadRunnerServiceStats { long lastRunTime = 0, beginTime = 0, totalTime = 0, totalSquaredTime = 0, minTime = Long.MAX_VALUE, maxTime = 0 int runCount = 0, errorCount = 0 ArtifactTypeStats artifactTypeStats = new ArtifactTypeStats() + Map getMap() { + Map newMap = [lastRunTime:lastRunTime, beginTime:beginTime, totalTime:totalTime, + totalSquaredTime:totalSquaredTime, minTime:minTime, maxTime:maxTime, runCount:runCount, errorCount:errorCount] as Map + newMap.put("artifactTypeStats", ObjectUtilities.objectToMap(artifactTypeStats)) + return newMap + } + } + static class LoadRunnerServiceInfo extends LoadRunnerServiceStats { + String serviceName, parametersExpr + int targetThreads, runDelayMs, rampDelayMs, timeBinLength, timeBinsKeep + AtomicInteger currentThreads = new AtomicInteger(0) + Map lastResult = null + ConcurrentLinkedDeque timeBinList = new ConcurrentLinkedDeque<>() + ArrayList runFutures = new ArrayList<>() + ScheduledFuture rampFuture = null - LoadRunnerServiceInfo(String serviceName, int targetThreads, int runDelayMs, int rampDelayMs) { - this.serviceName = serviceName + LoadRunnerServiceInfo(String serviceName, String parametersExpr, int targetThreads, + int runDelayMs, int rampDelayMs, int timeBinLength, int timeBinsKeep) { + this.serviceName = serviceName; this.parametersExpr = parametersExpr this.targetThreads = targetThreads - this.runDelayMs = runDelayMs - this.rampDelayMs = rampDelayMs + this.runDelayMs = runDelayMs; this.rampDelayMs = rampDelayMs + this.timeBinLength = timeBinLength; this.timeBinsKeep = timeBinsKeep } + + void countRun(LoadRunner loadRunner, long startTime, long endTime, ArtifactTypeStats stats) { + long runTime = endTime - startTime + // logger.info("count run ${serviceName} ${runTime} ${Thread.currentThread().name}") + LoadRunnerServiceStats curBin = null + + // find the current time bin in a semaphore locked section, the rest is increments and can be run multithreaded + loadRunner.mutateLock.lock() + // logger.info("count run after lock ${serviceName} ${runTime} ${Thread.currentThread().name}") + try { + if (beginTime == 0) beginTime = startTime + + curBin = timeBinList.isEmpty() ? null : timeBinList.getLast() + // create and add a new bin if there are none or if this hit is after the bin's end time (need to advance the bin) + if (curBin == null || curBin.beginTime + timeBinLength < startTime) { + curBin = new LoadRunnerServiceStats() + curBin.beginTime = startTime + timeBinList.add(curBin) + if (timeBinList.size() > timeBinsKeep) { + LoadRunnerServiceStats removeBin = timeBinList.removeFirst() + // logger.info("Removed time bin starting ${new Timestamp(removeBin.beginTime)} count ${removeBin.runCount}") + } + } + + // some exceptions, these are multiple operations and need to be in the locked section to be accurate, maybe don't need to be... + if (runTime < this.minTime) this.minTime = runTime + if (runTime > this.maxTime) this.maxTime = runTime + if (runTime < curBin.minTime) curBin.minTime = runTime + if (runTime > curBin.maxTime) curBin.maxTime = runTime + } finally { + loadRunner.mutateLock.unlock() + // logger.info("count run after unlock ${serviceName} ${runTime} ${Thread.currentThread().name}") + // loadRunner.logFutures() + } + + // for all runs + this.runCount++ + this.lastRunTime = endTime + this.totalTime += runTime + this.totalSquaredTime += runTime * runTime + this.artifactTypeStats.add(stats) + + // same thing for just this bin + curBin.runCount++ + curBin.lastRunTime = endTime + curBin.totalTime += runTime + curBin.totalSquaredTime += runTime * runTime + curBin.artifactTypeStats.add(stats) + } + void addThread(LoadRunner loadRunner) { - LoadRunnerServiceRunnable runnable = new LoadRunnerServiceRunnable(serviceName, loadRunner, this) + LoadRunnerServiceRunnable runnable = new LoadRunnerServiceRunnable(serviceName, parametersExpr, loadRunner) // NOTE: use scheduleWithFixedDelay so delay is wait between terminate of one and start of another, better for both short and long running test runs - loadRunner.scheduledExecutor.scheduleWithFixedDelay(runnable, 1, runDelayMs, TimeUnit.MILLISECONDS) + ScheduledFuture future = loadRunner.scheduledExecutor.scheduleWithFixedDelay(runnable, 1, runDelayMs, TimeUnit.MILLISECONDS) + // logger.info("Added run thread runDelayMs ${runDelayMs} ${runDelayMs?.class?.name} done ${future.done} ${future.toString()}") + runFutures.add(future) } void addRampThread(LoadRunner loadRunner) { + if (rampFuture != null && !rampFuture) beginTime = System.currentTimeMillis() LoadRunnerRamperRunnable runnable = new LoadRunnerRamperRunnable(loadRunner, this) // NOTE: use scheduleAtFixedRate so one is added each delay period regardless of how long it takes (generally not long) - loadRunner.scheduledExecutor.scheduleAtFixedRate(runnable, 1, rampDelayMs, TimeUnit.MILLISECONDS) + rampFuture = loadRunner.scheduledExecutor.scheduleAtFixedRate(runnable, 1, rampDelayMs, TimeUnit.MILLISECONDS) } void resetStats() { lastRunTime = 0; beginTime = 0; totalTime = 0; totalSquaredTime = 0; minTime = Long.MAX_VALUE; maxTime = 0 runCount = 0; errorCount = 0 artifactTypeStats = new ArtifactTypeStats() lastResult = null + timeBinList = new ConcurrentLinkedDeque<>() } } static class LoadRunnerRamperRunnable implements Runnable { @@ -738,6 +829,12 @@ class ServiceFacadeImpl implements ServiceFacade { // TODO add delayed ramp-down, useful for some performance behavior patterns but usually redundant with delayed ramp up to look for elbows in the response time over time } } + static class LoadRunnerThreadFactory implements ThreadFactory { + private final ThreadGroup workerGroup = new ThreadGroup("LoadRunner") + private final AtomicInteger threadNumber = new AtomicInteger(1) + Thread newThread(Runnable r) { return new Thread(workerGroup, r, "LoadRunner-" + threadNumber.getAndIncrement()) } + } + static class LoadRunner { ExecutionContextFactoryImpl ecfi CustomScheduledExecutor scheduledExecutor = null @@ -750,17 +847,22 @@ class ServiceFacadeImpl implements ServiceFacade { this.ecfi = ecfi } - void setServiceInfo(String serviceName, String parametersExpr, int targetThreads, int runDelayMs, int rampDelayMs) { + LoadRunnerServiceInfo getServiceInfo(String serviceName, String parametersExpr) { + for (int i = 0; i < serviceInfos.size(); i++) { + LoadRunnerServiceInfo curInfo = (LoadRunnerServiceInfo) serviceInfos.get(i) + if (curInfo.serviceName == serviceName && curInfo.parametersExpr == parametersExpr) + return curInfo + } + return null + } + void setServiceInfo(String serviceName, String parametersExpr, int targetThreads, int runDelayMs, + int rampDelayMs, int timeBinLength, int timeBinsKeep) { mutateLock.lock() try { - LoadRunnerServiceInfo serviceInfo = null - for (int i = 0; i < serviceInfos.size(); i++) { - LoadRunnerServiceInfo curInfo = (LoadRunnerServiceInfo) serviceInfos.get(i) - if (curInfo.serviceName == serviceName) serviceInfo = curInfo - } + LoadRunnerServiceInfo serviceInfo = getServiceInfo(serviceName, parametersExpr) if (serviceInfo == null) { - serviceInfo = new LoadRunnerServiceInfo(serviceName, targetThreads, runDelayMs, rampDelayMs) - serviceInfo.parametersExpr = parametersExpr + serviceInfo = new LoadRunnerServiceInfo(serviceName, parametersExpr, targetThreads, + runDelayMs, rampDelayMs, timeBinLength, timeBinsKeep) serviceInfos.add(serviceInfo) @@ -769,10 +871,11 @@ class ServiceFacadeImpl implements ServiceFacade { serviceInfo.addRampThread(this) } } else { - serviceInfo.parametersExpr = parametersExpr serviceInfo.targetThreads = targetThreads serviceInfo.runDelayMs = runDelayMs serviceInfo.rampDelayMs = rampDelayMs + serviceInfo.timeBinLength = timeBinLength + serviceInfo.timeBinsKeep = timeBinsKeep } } finally { mutateLock.unlock() @@ -793,14 +896,12 @@ class ServiceFacadeImpl implements ServiceFacade { curInfo.resetStats() } - scheduledExecutor = new CustomScheduledExecutor(corePoolSize) + scheduledExecutor = new CustomScheduledExecutor(corePoolSize, new LoadRunnerThreadFactory()) scheduledExecutor.setMaximumPoolSize(maxPoolSize) for (int i = 0; i < serviceInfos.size(); i++) { LoadRunnerServiceInfo curInfo = (LoadRunnerServiceInfo) serviceInfos.get(i) - // do this once here - curInfo.addThread(this) - // add a schedule for the rest + // get the ramp thread started curInfo.addRampThread(this) } } @@ -838,5 +939,16 @@ class ServiceFacadeImpl implements ServiceFacade { mutateLock.unlock() } } + + void logFutures() { + for (int si = 0; si < serviceInfos.size(); si++) { + LoadRunnerServiceInfo serviceInfo = serviceInfos.get(si) + logger.info("LoadRunner RAMP Future done ${serviceInfo.rampFuture?.done} canceled ${serviceInfo.rampFuture?.cancelled} ${serviceInfo.rampFuture?.toString()}") + for (int i = 0; i < serviceInfo.runFutures.size(); i++) { + ScheduledFuture future = serviceInfo.runFutures.get(i) + logger.info("LoadRunner RUN Future done ${future.done} canceled ${future.cancelled} ${future.toString()}") + } + } + } } } diff --git a/framework/src/main/java/org/moqui/util/ObjectUtilities.java b/framework/src/main/java/org/moqui/util/ObjectUtilities.java index f2b794603..16a9c2754 100644 --- a/framework/src/main/java/org/moqui/util/ObjectUtilities.java +++ b/framework/src/main/java/org/moqui/util/ObjectUtilities.java @@ -76,6 +76,7 @@ public static Map objectToMap(Object bean) { } } + /* commenting for now, seems to call a bunch of undesired methods, will need some work to filter them out: try { BeanInfo beanInfo = Introspector.getBeanInfo(clazz); PropertyDescriptor[] propertyDescriptors = beanInfo.getPropertyDescriptors(); @@ -93,6 +94,7 @@ public static Map objectToMap(Object bean) { } catch (IntrospectionException e) { // nothing again } + */ // this gets picked up automatically, just remove at the end, faster than checking along the way map.remove("class"); From d3ddb8b97096679ba7627f79d387b15f30574363 Mon Sep 17 00:00:00 2001 From: David E Jones Date: Sun, 5 Feb 2023 17:27:46 -0800 Subject: [PATCH 08/26] In Service LoadRunner add small random delay support, use available threads * 4 for max load pool size; add permission for SERVICE_LOAD_RUNNER for screens --- framework/data/SecurityTypeData.xml | 2 ++ .../impl/service/ServiceFacadeImpl.groovy | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/framework/data/SecurityTypeData.xml b/framework/data/SecurityTypeData.xml index 0906d9648..33544305b 100644 --- a/framework/data/SecurityTypeData.xml +++ b/framework/data/SecurityTypeData.xml @@ -54,4 +54,6 @@ along with this software (see the LICENSE.md file). If not, see + + diff --git a/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy index b6c5fb52f..da7bf9357 100644 --- a/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/service/ServiceFacadeImpl.groovy @@ -691,6 +691,10 @@ class ServiceFacadeImpl implements ServiceFacade { } } + // before starting, and tracking the startTime, do a small random delay for variation in run times + if (serviceInfo.runDelayVaryMs != 0) + Thread.sleep(ThreadLocalRandom.current().nextInt(serviceInfo.runDelayVaryMs)) + long startTime = System.currentTimeMillis() ExecutionContextImpl threadEci = ecfi.getEci() try { @@ -699,7 +703,8 @@ class ServiceFacadeImpl implements ServiceFacade { // run the service try { - serviceInfo.lastResult = threadEci.serviceFacade.sync().name(serviceName).parameters(parameters).disableAuthz().call() + serviceInfo.lastResult = threadEci.serviceFacade.sync().name(serviceName) + .parameters(parameters).disableAuthz().call() } catch (Throwable t) { // logged elsewhere, just count and swallow serviceInfo.errorCount++ @@ -726,7 +731,7 @@ class ServiceFacadeImpl implements ServiceFacade { } static class LoadRunnerServiceInfo extends LoadRunnerServiceStats { String serviceName, parametersExpr - int targetThreads, runDelayMs, rampDelayMs, timeBinLength, timeBinsKeep + int targetThreads, runDelayMs, runDelayVaryMs, rampDelayMs, timeBinLength, timeBinsKeep AtomicInteger currentThreads = new AtomicInteger(0) Map lastResult = null @@ -735,10 +740,10 @@ class ServiceFacadeImpl implements ServiceFacade { ScheduledFuture rampFuture = null LoadRunnerServiceInfo(String serviceName, String parametersExpr, int targetThreads, - int runDelayMs, int rampDelayMs, int timeBinLength, int timeBinsKeep) { + int runDelayMs, int runDelayVaryMs, int rampDelayMs, int timeBinLength, int timeBinsKeep) { this.serviceName = serviceName; this.parametersExpr = parametersExpr this.targetThreads = targetThreads - this.runDelayMs = runDelayMs; this.rampDelayMs = rampDelayMs + this.runDelayMs = runDelayMs; this.runDelayVaryMs = runDelayVaryMs; this.rampDelayMs = rampDelayMs this.timeBinLength = timeBinLength; this.timeBinsKeep = timeBinsKeep } @@ -839,7 +844,7 @@ class ServiceFacadeImpl implements ServiceFacade { ExecutionContextFactoryImpl ecfi CustomScheduledExecutor scheduledExecutor = null ArrayList serviceInfos = new ArrayList<>() - int corePoolSize = 4, maxPoolSize = 8 + Integer corePoolSize = 4, maxPoolSize = null AtomicInteger execIndex = new AtomicInteger(1) ReentrantLock mutateLock = new ReentrantLock() @@ -856,13 +861,13 @@ class ServiceFacadeImpl implements ServiceFacade { return null } void setServiceInfo(String serviceName, String parametersExpr, int targetThreads, int runDelayMs, - int rampDelayMs, int timeBinLength, int timeBinsKeep) { + int runDelayVaryMs, int rampDelayMs, int timeBinLength, int timeBinsKeep) { mutateLock.lock() try { LoadRunnerServiceInfo serviceInfo = getServiceInfo(serviceName, parametersExpr) if (serviceInfo == null) { serviceInfo = new LoadRunnerServiceInfo(serviceName, parametersExpr, targetThreads, - runDelayMs, rampDelayMs, timeBinLength, timeBinsKeep) + runDelayMs, runDelayVaryMs, rampDelayMs, timeBinLength, timeBinsKeep) serviceInfos.add(serviceInfo) @@ -884,7 +889,6 @@ class ServiceFacadeImpl implements ServiceFacade { void begin() { mutateLock.lock() try { - // TODO set maxPoolSize to CPU count x2 or something if needed to scale the load runner itself; probably not much as services running on same system... if (scheduledExecutor == null) { // restart index execIndex = new AtomicInteger(1) @@ -897,6 +901,7 @@ class ServiceFacadeImpl implements ServiceFacade { } scheduledExecutor = new CustomScheduledExecutor(corePoolSize, new LoadRunnerThreadFactory()) + if (maxPoolSize == null) maxPoolSize = Runtime.getRuntime().availableProcessors() * 4 scheduledExecutor.setMaximumPoolSize(maxPoolSize) for (int i = 0; i < serviceInfos.size(); i++) { From e462fed896f84761361ab3f143eef7b0bc19eac7 Mon Sep 17 00:00:00 2001 From: David E Jones Date: Mon, 6 Feb 2023 15:19:16 -0800 Subject: [PATCH 09/26] In build.gradle also delete the SaveOpenSearch.zip file in cleanLoadSave, also called by cleanAll, was missing in the prior OpenSearch changes --- build.gradle | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 1ab9e682b..3b591d4c6 100644 --- a/build.gradle +++ b/build.gradle @@ -75,8 +75,9 @@ task cleanDb { doLast { } } task cleanLog(type: Delete) { delete fileTree(dir: moquiRuntime+'/log', include: '*') } task cleanSessions(type: Delete) { delete fileTree(dir: moquiRuntime+'/sessions', include: '*') } -task cleanLoadSave(type: Delete) { delete file('SaveH2.zip'); delete file('SaveDEFAULT.zip'); - delete file('SaveTransactional.zip'); delete file('SaveAnalytical.zip'); delete file('SaveOrientDb.zip'); delete file('SaveElasticSearch.zip') } +task cleanLoadSave(type: Delete) { delete file('SaveH2.zip'); delete file('SaveDEFAULT.zip') + delete file('SaveTransactional.zip'); delete file('SaveAnalytical.zip'); delete file('SaveOrientDb.zip') + delete file('SaveElasticSearch.zip'); delete file('SaveOpenSearch.zip') } task cleanPlusRuntime(type: Delete) { delete file(plusRuntimeName) } task cleanOther(type: Delete) { delete fileTree(dir: '.', includes: ['**/.nbattrs', '**/*~', '**/.#*', '**/.DS_Store', '**/*.rej', '**/*.orig']) } From 5bd110b1982bb58518da3e61b3cebf446aa2b0a1 Mon Sep 17 00:00:00 2001 From: David E Jones Date: Mon, 6 Feb 2023 15:22:16 -0800 Subject: [PATCH 10/26] Fix PIT testing against ElasticSearch 7.10.2, the last open source version; the docs are horrible with 3 differences between what actually ended working from a bunch of random experimentation; PIT support is necessary for the Elastic Entity implementation (alternative to a DB cursor); this ended up being needed because more recent OpenSearch that supports PIT also uses more memory than ElasticSearch, enough that the anemic demo.moqui.org server with 2GB RAM can't run Moqui + OpenSearch 2.4 --- .../org/moqui/impl/context/ElasticFacadeImpl.groovy | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy index 2d30bdd1d..d9971b5ef 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy @@ -493,8 +493,14 @@ class ElasticFacadeImpl implements ElasticFacade { // requires 2.4.0 or later response = makeRestClient(Method.POST, index, "_search/point_in_time", [keep_alive:keepAlive]).call() } else { - // see: https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#scroll-search-results - response = makeRestClient(Method.POST, index, "_pit", [keep_alive:keepAlive]).call() + // see: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/paginate-search-results.html#scroll-search-results + // whatever the docs say: + // - it doesn't work with the keep_alive parameter at all "contains unrecognized parameter: [keep_alive]" + // - does not work with no body "request body is required" + // - and it doesn't work without the doc type _doc before _pit in the path "mapping type name [_pit] can't start with '_' unless it is called [_doc]" + // in other words, the docs are completely wrong for ES 7.10.2 + // response = makeRestClient(Method.POST, index, "_pit", [keep_alive:keepAlive]).call() + response = makeRestClient(Method.POST, index, "_doc/_pit", null).text(objectToJson([keep_alive:keepAlive])).call() } // System.out.println("Get PIT Response: ${response.statusCode} ${response.reasonPhrase}\n${response.text()}") checkResponse(response, "PIT", index) @@ -510,7 +516,7 @@ class ElasticFacadeImpl implements ElasticFacade { response = makeRestClient(Method.DELETE, null, "_search/point_in_time", null) .text(objectToJson([pit_id:[pitId]])).call() } else { - // see: https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#scroll-search-results + // see: https://www.elastic.co/guide/en/elasticsearch/reference/7.10/paginate-search-results.html#scroll-search-results response = makeRestClient(Method.DELETE, null, "_pit", null).text(objectToJson([id:pitId])).call() } // System.out.println("Delete PIT Response: ${response.statusCode} ${response.reasonPhrase}\n${response.text()}") From 87add5973862d6c6dfeb9c6eba2ed2849142a31a Mon Sep 17 00:00:00 2001 From: David E Jones Date: Mon, 6 Feb 2023 15:53:47 -0800 Subject: [PATCH 11/26] Add new moqui-demo component repo to addons.xml --- addons.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/addons.xml b/addons.xml index e5a926a14..8e7e0e3de 100644 --- a/addons.xml +++ b/addons.xml @@ -77,7 +77,9 @@ + + From f88210ed4a28cc40a6ea6b1aacd55b63b4b17d72 Mon Sep 17 00:00:00 2001 From: Acetousk Date: Wed, 15 Feb 2023 11:24:00 -0700 Subject: [PATCH 12/26] Add getScreenPathHasTransition in ScreenRenderImpl --- .../org/moqui/impl/screen/ScreenRenderImpl.groovy | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy b/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy index f3e8b8ee6..183c22cfd 100644 --- a/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy @@ -1105,6 +1105,19 @@ class ScreenRenderImpl implements ScreenRender { return activePath } + // TODO: This may not be the actual place we decided on, but due to lost work this is my best guess + // Get the first screen path of the parent screens with a transition specified of the currently rendered screen + String getScreenPathHasTransition(String transitionName) { + int screenPathDefListSize = screenUrlInfo.screenPathDefList.size() + for (int i = 0; i < screenPathDefListSize; i++) { + ScreenDefinition screenDef = (ScreenDefinition) screenUrlInfo.screenPathDefList.get(i) + if (screenDef.hasTransition(transitionName)) { + return '/' + screenUrlInfo.fullPathNameList.subList(0,i).join('/') + (i == 0 ? '' : '/') + } + } + return null + } + String renderSubscreen() { // first see if there is another screen def in the list if (!getActiveScreenHasNext()) { From abe9936f3051eff2c72f3a4c0e4112cc84a0e0ce Mon Sep 17 00:00:00 2001 From: David E Jones Date: Sun, 19 Feb 2023 13:50:48 -0800 Subject: [PATCH 13/26] In ElasticDatasourceFactory add better exceptions than NPE when no ElasticClient is found --- .../impl/entity/elastic/ElasticDatasourceFactory.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/framework/src/main/groovy/org/moqui/impl/entity/elastic/ElasticDatasourceFactory.groovy b/framework/src/main/groovy/org/moqui/impl/entity/elastic/ElasticDatasourceFactory.groovy index 4567832c8..a532332c0 100644 --- a/framework/src/main/groovy/org/moqui/impl/entity/elastic/ElasticDatasourceFactory.groovy +++ b/framework/src/main/groovy/org/moqui/impl/entity/elastic/ElasticDatasourceFactory.groovy @@ -161,6 +161,7 @@ class ElasticDatasourceFactory implements EntityDatasourceFactory { if (checkedEntityIndexSet.contains(indexName)) return ElasticFacade.ElasticClient elasticClient = efi.ecfi.elasticFacade.getClient(clusterName) + if (elasticClient == null) throw new IllegalStateException("No ElasticClient found for cluster name " + clusterName) if (!elasticClient.indexExists(indexName)) { Map mapping = makeElasticEntityMapping(ed) // logger.warn("Creating ES Index ${indexName} with mapping: ${JsonOutput.prettyPrint(JsonOutput.toJson(mapping))}") @@ -170,7 +171,11 @@ class ElasticDatasourceFactory implements EntityDatasourceFactory { checkedEntityIndexSet.add(indexName) } - ElasticFacade.ElasticClient getElasticClient() { efi.ecfi.elasticFacade.getClient(clusterName) } + ElasticFacade.ElasticClient getElasticClient() { + ElasticFacade.ElasticClient client = efi.ecfi.elasticFacade.getClient(clusterName) + if (client == null) throw new IllegalStateException("No ElasticClient found for cluster name " + clusterName) + return client + } String getIndexName(EntityDefinition ed) { return indexPrefix + ed.getTableNameLowerCase() } From 185a6966b39b5fd9452757f43814381c9e766cba Mon Sep 17 00:00:00 2001 From: David E Jones Date: Sun, 19 Feb 2023 21:33:44 -0800 Subject: [PATCH 14/26] Increase default worker-pool-core to 16 threads, and max to 32 or CPU * 3 threads (instead of *2); increasing after tests and production results showing typical low CPU time per thread (much of it is deferred database or search with lots of I/O wait); also improve logging when waiting for worker pool to empty; note that this also improves automated test performance because many of the tests get to hundreds of async services queued up and the ThreadPoolExecutor seems to add threads slowly even if the pool is fully utilized --- .../moqui/impl/context/ExecutionContextFactoryImpl.groovy | 8 ++++---- framework/src/main/resources/MoquiDefaultConf.xml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/ExecutionContextFactoryImpl.groovy b/framework/src/main/groovy/org/moqui/impl/context/ExecutionContextFactoryImpl.groovy index f42d6f8f2..e6c93a26f 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ExecutionContextFactoryImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/ExecutionContextFactoryImpl.groovy @@ -452,10 +452,10 @@ class ExecutionContextFactoryImpl implements ExecutionContextFactory { BlockingQueue workQueue = new LinkedBlockingQueue<>(workerQueueSize) int coreSize = (toolsNode.attribute("worker-pool-core") ?: "16") as int - int maxSize = (toolsNode.attribute("worker-pool-max") ?: "24") as int - int availableProcessorsSize = Runtime.getRuntime().availableProcessors() * 2 + int maxSize = (toolsNode.attribute("worker-pool-max") ?: "32") as int + int availableProcessorsSize = Runtime.getRuntime().availableProcessors() * 3 if (availableProcessorsSize > maxSize) { - logger.info("Setting worker pool size to ${availableProcessorsSize} based on available processors * 2") + logger.info("Setting worker pool size to ${availableProcessorsSize} based on available processors * 3") maxSize = availableProcessorsSize } long aliveTime = (toolsNode.attribute("worker-pool-alive") ?: "60") as long @@ -467,9 +467,9 @@ class ExecutionContextFactoryImpl implements ExecutionContextFactory { boolean waitWorkerPoolEmpty(int retryLimit) { ThreadPoolExecutor jobWorkerPool = serviceFacade.jobWorkerPool int count = 0 - logger.warn("Wait for workerPool and jobWorkerPool empty: worker queue size ${workerPool.getQueue().size()} active ${workerPool.getActiveCount()}; service job queue size ${jobWorkerPool.getQueue().size()} active ${jobWorkerPool.getActiveCount()}") while (count < retryLimit && (workerPool.getQueue().size() > 0 || workerPool.getActiveCount() > 0 || jobWorkerPool.getQueue().size() > 0 || jobWorkerPool.getActiveCount() > 0)) { + if (count % 10 == 0) logger.warn("Wait for workerPool and jobWorkerPool empty: worker queue size ${workerPool.getQueue().size()} active ${workerPool.getActiveCount()} max threads ${workerPool.getMaximumPoolSize()}; service job queue size ${jobWorkerPool.getQueue().size()} active ${jobWorkerPool.getActiveCount()}") Thread.sleep(100) count++ } diff --git a/framework/src/main/resources/MoquiDefaultConf.xml b/framework/src/main/resources/MoquiDefaultConf.xml index d2ad0edee..e44aeb3f3 100644 --- a/framework/src/main/resources/MoquiDefaultConf.xml +++ b/framework/src/main/resources/MoquiDefaultConf.xml @@ -81,7 +81,7 @@ - + From b17d351f19b2299ce7871538c107bcc94dcf7b5e Mon Sep 17 00:00:00 2001 From: aabiabdallah Date: Tue, 21 Feb 2023 17:35:01 -0600 Subject: [PATCH 15/26] In ScreenRenderImpl remove token created requirement when checking for session token --- .../main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy b/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy index 183c22cfd..71926282f 100644 --- a/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy @@ -425,7 +425,6 @@ class ScreenRenderImpl implements ScreenRender { // require a moquiSessionToken parameter for all but get if (request.getMethod().toLowerCase() != "get" && webappInfo != null && webappInfo.requireSessionToken && targetTransition.getRequireSessionToken() && - !"true".equals(request.getAttribute("moqui.session.token.created")) && !"true".equals(request.getAttribute("moqui.request.authenticated"))) { String passedToken = (String) ec.web.getParameters().get("moquiSessionToken") if (!passedToken) passedToken = request.getHeader("moquiSessionToken") ?: From c7b038fc23f2cfc823bcd6a8455987ff36251590 Mon Sep 17 00:00:00 2001 From: "David E. Jones" Date: Tue, 21 Feb 2023 17:25:14 -0800 Subject: [PATCH 16/26] Revert "Update to session token condition" --- .../main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy b/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy index 71926282f..183c22cfd 100644 --- a/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/screen/ScreenRenderImpl.groovy @@ -425,6 +425,7 @@ class ScreenRenderImpl implements ScreenRender { // require a moquiSessionToken parameter for all but get if (request.getMethod().toLowerCase() != "get" && webappInfo != null && webappInfo.requireSessionToken && targetTransition.getRequireSessionToken() && + !"true".equals(request.getAttribute("moqui.session.token.created")) && !"true".equals(request.getAttribute("moqui.request.authenticated"))) { String passedToken = (String) ec.web.getParameters().get("moquiSessionToken") if (!passedToken) passedToken = request.getHeader("moquiSessionToken") ?: From 1b7d7e148cdbd3bc18b6d55de3aaf92970132ed8 Mon Sep 17 00:00:00 2001 From: Wei Zhang Date: Thu, 23 Feb 2023 07:03:27 +0800 Subject: [PATCH 17/26] Fixed a runtime error if Currency is BTC (#555) * Fixed the problem that moqui cannot be deployed as non-root webapp in Tomcat * Fixed a runtime error if Currency is BTC * Fixed a runtime error if Currency is BTC * Fixed the retries of Elastic Client --- AUTHORS | 2 ++ .../org/moqui/impl/context/ElasticFacadeImpl.groovy | 2 +- .../groovy/org/moqui/impl/context/L10nFacadeImpl.java | 10 +++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 566d88618..27dbd1a34 100644 --- a/AUTHORS +++ b/AUTHORS @@ -60,6 +60,7 @@ Written in 2020 by Jacob Barnes - Tellan Written in 2020 by Amir Anjomshoaa - amiranjom Written in 2021 by Deepak Dixit - dixitdeepak Written in 2021 by Taher Alkhateeb - pythys +Written in 2022 by Zhang Wei - hellozhangwei =========================================================================== @@ -106,5 +107,6 @@ Written in 2020 by Jacob Barnes - Tellan Written in 2020 by Amir Anjomshoaa - amiranjom Written in 2021 by Deepak Dixit - dixitdeepak Written in 2021 by Taher Alkhateeb - pythys +Written in 2022 by Zhang Wei - hellozhangwei =========================================================================== diff --git a/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy index d9971b5ef..3155ec965 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy @@ -196,7 +196,7 @@ class ElasticFacadeImpl implements ElasticFacade { requestFactory.init() // try connecting and get server info - int retries = clusterHost == 'localhost' && !"true".equals(System.getProperty("moqui.elasticsearch.started")) ? 1 : 20 + int retries = ((clusterHost == 'localhost' || clusterHost == '127.0.0.1') && !"true".equals(System.getProperty("moqui.elasticsearch.started"))) ? 1 : 20 for (int i = 1; i <= retries; i++) { try { serverInfo = getServerInfo() diff --git a/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java b/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java index 65b4511b9..acec7f13f 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java +++ b/framework/src/main/groovy/org/moqui/impl/context/L10nFacadeImpl.java @@ -116,7 +116,15 @@ public String formatCurrency(Object amount, String uomId, Integer fractionDigits } } - Currency currency = uomId != null && uomId.length() > 0 ? Currency.getInstance(uomId) : null; + Currency currency = null; + if (uomId != null && uomId.length() > 0) { + try { + currency = Currency.getInstance(uomId); + } catch (Exception e) { + if (logger.isTraceEnabled()) logger.trace("Ignoring IllegalArgumentException for Currency parse: " + e.toString()); + } + } + if (locale == null) locale = getLocale(); if (currency != null) { NumberFormat nf = NumberFormat.getCurrencyInstance(locale); From 4478e3a5e4828e7d6def2c617ab2998a5640fe12 Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Wed, 22 Feb 2023 20:22:34 -0300 Subject: [PATCH 18/26] Anonymous usage of screens with transitions (#541) * fix: consider transitions in screens or parent screens defined with require-authentication in "anonymous-view" or "anonymous-all" as permitted * fix: consider transitions defined with authenticate in "anonymous-view" or "anonymous-all" as permitted * fix: consider any of service.@authenticate and screen.@require-authentication to determine permission on transition * fix: considering verb-based execution actions of single-service when determining transition permission, correctly determining whether view or all allowed by screen --- .../moqui/impl/screen/ScreenUrlInfo.groovy | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/screen/ScreenUrlInfo.groovy b/framework/src/main/groovy/org/moqui/impl/screen/ScreenUrlInfo.groovy index 3ce33c96e..ada1939f4 100644 --- a/framework/src/main/groovy/org/moqui/impl/screen/ScreenUrlInfo.groovy +++ b/framework/src/main/groovy/org/moqui/impl/screen/ScreenUrlInfo.groovy @@ -271,6 +271,9 @@ class ScreenUrlInfo { ArrayDeque artifactExecutionInfoStack = new ArrayDeque() int screenPathDefListSize = screenPathDefList.size() + boolean allowedByScreenDefinitionView = false + boolean allowedByScreenDefinitionAll = false + boolean allowedByScreenDefinition = false for (int i = 0; i < screenPathDefListSize; i++) { AuthzAction curActionEnum = (i == (screenPathDefListSize - 1)) ? actionEnum : ArtifactExecutionInfo.AUTHZA_VIEW ScreenDefinition screenDef = (ScreenDefinition) screenPathDefList.get(i) @@ -285,9 +288,15 @@ class ScreenUrlInfo { MNode screenNode = screenDef.getScreenNode() String requireAuthentication = screenNode.attribute('require-authentication') + allowedByScreenDefinitionView = "anonymous-view".equals(requireAuthentication) + allowedByScreenDefinitionAll = "anonymous-all".equals(requireAuthentication) + if (actionEnum == ArtifactExecutionInfo.AUTHZA_VIEW) { + allowedByScreenDefinition = allowedByScreenDefinition || allowedByScreenDefinitionView || allowedByScreenDefinitionAll + } else if (actionEnum == ArtifactExecutionInfo.AUTHZA_ALL) + allowedByScreenDefinition = allowedByScreenDefinition || allowedByScreenDefinitionAll if (!aefi.isPermitted(aeii, lastAeii, isLast ? (!requireAuthentication || "true".equals(requireAuthentication)) : false, false, false, artifactExecutionInfoStack)) { - // logger.warn("TOREMOVE user ${username} is NOT allowed to view screen at path ${this.fullPathNameList} because of screen at ${screenDef.location}") + //logger.warn("TOREMOVE user ${userId} is NOT allowed to view screen at path ${this.fullPathNameList} because of screen at ${screenDef.location}") if (permittedCacheKey != null) aefi.screenPermittedCache.put(permittedCacheKey, false) return false } @@ -296,7 +305,7 @@ class ScreenUrlInfo { } // see if the transition is permitted - if (transitionItem != null) { + if (!allowedByScreenDefinition && transitionItem != null) { ScreenDefinition lastScreenDef = (ScreenDefinition) screenPathDefList.get(screenPathDefList.size() - 1) ArtifactExecutionInfoImpl aeii = new ArtifactExecutionInfoImpl("${lastScreenDef.location}/${transitionItem.name}", ArtifactExecutionInfo.AT_XML_SCREEN_TRANS, ArtifactExecutionInfo.AUTHZA_VIEW, null) @@ -317,10 +326,15 @@ class ScreenUrlInfo { if (authzAction == null) authzAction = ServiceDefinition.verbAuthzActionEnumMap.get(ServiceDefinition.getVerbFromName(serviceName)) if (authzAction == null) authzAction = ArtifactExecutionInfo.AUTHZA_ALL + boolean allowedByServiceDefinition = false + if (authzAction == ArtifactExecutionInfo.AUTHZA_VIEW) { + allowedByServiceDefinition = allowedByScreenDefinitionView || "anonymous-view".equals(sd.authenticate) || "anonymous-all".equals(sd.authenticate) + } else if (authzAction in [ArtifactExecutionInfo.AUTHZA_ALL, ArtifactExecutionInfo.AUTHZA_CREATE, ArtifactExecutionInfo.AUTHZA_UPDATE, ArtifactExecutionInfo.AUTHZA_DELETE]) + allowedByServiceDefinition = allowedByScreenDefinitionAll || "anonymous-all".equals(sd.authenticate) ArtifactExecutionInfoImpl aeii = new ArtifactExecutionInfoImpl(serviceName, ArtifactExecutionInfo.AT_SERVICE, authzAction, null) ArtifactExecutionInfoImpl lastAeii = (ArtifactExecutionInfoImpl) artifactExecutionInfoStack.peekFirst() - if (!aefi.isPermitted(aeii, lastAeii, true, false, false, null)) { + if (!aefi.isPermitted(aeii, lastAeii, !allowedByServiceDefinition, false, false, null)) { // logger.warn("TOREMOVE user ${username} is NOT allowed to run transition at path ${this.fullPathNameList} because of screen at ${screenDef.location}") if (permittedCacheKey != null) aefi.screenPermittedCache.put(permittedCacheKey, false) return false From 4f858cbc410238151dcf610f1260a0f8cd5d9ad5 Mon Sep 17 00:00:00 2001 From: David E Jones Date: Thu, 23 Feb 2023 02:25:10 -0800 Subject: [PATCH 19/26] Small fix for new authz changes for anonymous-view/-all, handle no screen definition instead of NPE --- .../main/groovy/org/moqui/impl/screen/ScreenUrlInfo.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/screen/ScreenUrlInfo.groovy b/framework/src/main/groovy/org/moqui/impl/screen/ScreenUrlInfo.groovy index ada1939f4..ec31ef405 100644 --- a/framework/src/main/groovy/org/moqui/impl/screen/ScreenUrlInfo.groovy +++ b/framework/src/main/groovy/org/moqui/impl/screen/ScreenUrlInfo.groovy @@ -328,9 +328,10 @@ class ScreenUrlInfo { boolean allowedByServiceDefinition = false if (authzAction == ArtifactExecutionInfo.AUTHZA_VIEW) { - allowedByServiceDefinition = allowedByScreenDefinitionView || "anonymous-view".equals(sd.authenticate) || "anonymous-all".equals(sd.authenticate) - } else if (authzAction in [ArtifactExecutionInfo.AUTHZA_ALL, ArtifactExecutionInfo.AUTHZA_CREATE, ArtifactExecutionInfo.AUTHZA_UPDATE, ArtifactExecutionInfo.AUTHZA_DELETE]) - allowedByServiceDefinition = allowedByScreenDefinitionAll || "anonymous-all".equals(sd.authenticate) + allowedByServiceDefinition = allowedByScreenDefinitionView || (sd != null && ("anonymous-view".equals(sd.authenticate) || "anonymous-all".equals(sd.authenticate))) + } else if (authzAction in [ArtifactExecutionInfo.AUTHZA_ALL, ArtifactExecutionInfo.AUTHZA_CREATE, ArtifactExecutionInfo.AUTHZA_UPDATE, ArtifactExecutionInfo.AUTHZA_DELETE]) { + allowedByServiceDefinition = allowedByScreenDefinitionAll || (sd != null && "anonymous-all".equals(sd.authenticate)) + } ArtifactExecutionInfoImpl aeii = new ArtifactExecutionInfoImpl(serviceName, ArtifactExecutionInfo.AT_SERVICE, authzAction, null) ArtifactExecutionInfoImpl lastAeii = (ArtifactExecutionInfoImpl) artifactExecutionInfoStack.peekFirst() From a97e5ed420caad29ce8ee3e2fa4d3dc7f650a74d Mon Sep 17 00:00:00 2001 From: Jens Hardings Date: Thu, 23 Feb 2023 11:30:39 -0300 Subject: [PATCH 20/26] Handle usernames with different casing in session data --- .../groovy/org/moqui/impl/context/UserFacadeImpl.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/UserFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/context/UserFacadeImpl.groovy index 0a37ad7aa..abd9f7b23 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/UserFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/UserFacadeImpl.groovy @@ -113,8 +113,10 @@ class UserFacadeImpl implements UserFacade { // user found in session so no login needed, but make sure hasLoggedOut != "Y" EntityValue userAccount = (EntityValue) null if (sesUsername != null && !sesUsername.isEmpty()) { + EntityCondition usernameCond = eci.entityFacade.getConditionFactory() + .makeCondition("username", EntityCondition.ComparisonOperator.EQUALS, username).ignoreCase() userAccount = eci.getEntity().find("moqui.security.UserAccount") - .condition("username", sesUsername).useCache(false).disableAuthz().one() + .condition(usernameCond).useCache(false).disableAuthz().one() } if (userAccount != null && "Y".equals(userAccount.getNoCheckSimple("hasLoggedOut"))) { @@ -1051,8 +1053,10 @@ class UserFacadeImpl implements UserFacade { EntityValueBase ua = (EntityValueBase) null if (username != null && username.length() > 0) { + EntityCondition usernameCond = ufi.eci.entityFacade.getConditionFactory() + .makeCondition("username", EntityCondition.ComparisonOperator.EQUALS, username).ignoreCase() ua = (EntityValueBase) ufi.eci.getEntity().find("moqui.security.UserAccount") - .condition("username", username).useCache(false).disableAuthz().one() + .condition(usernameCond).useCache(false).disableAuthz().one() } if (ua != null) { userAccount = ua From 5e70c5080356c187da444821390170694dfe193a Mon Sep 17 00:00:00 2001 From: user Date: Tue, 7 Mar 2023 12:55:31 -0600 Subject: [PATCH 21/26] Add common java includes to the xml actions ftl file --- framework/template/XmlActions.groovy.ftl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/framework/template/XmlActions.groovy.ftl b/framework/template/XmlActions.groovy.ftl index 66a235934..39236c122 100644 --- a/framework/template/XmlActions.groovy.ftl +++ b/framework/template/XmlActions.groovy.ftl @@ -15,6 +15,8 @@ import static org.moqui.util.ObjectUtilities.* import static org.moqui.util.CollectionUtilities.* import static org.moqui.util.StringUtilities.* import java.sql.Timestamp +import java.sql.Time +import java.time.* // these are in the context by default: ExecutionContext ec, Map context, Map result <#visit xmlActionsRoot/> From f5e57d64585fe3627deff1fe111e63ad503b523f Mon Sep 17 00:00:00 2001 From: David E Jones Date: Tue, 7 Mar 2023 13:11:43 -0800 Subject: [PATCH 22/26] Add subTopic field to NotificationMessage --- framework/entity/SecurityEntities.xml | 1 + .../moqui/impl/context/NotificationMessageImpl.groovy | 11 +++++++++-- .../java/org/moqui/context/NotificationMessage.java | 3 +++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/framework/entity/SecurityEntities.xml b/framework/entity/SecurityEntities.xml index 71cc6405b..e5e7f8a26 100644 --- a/framework/entity/SecurityEntities.xml +++ b/framework/entity/SecurityEntities.xml @@ -513,6 +513,7 @@ along with this software (see the LICENSE.md file). If not, see + diff --git a/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy b/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy index b30563d2d..8c990c8c9 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy @@ -37,6 +37,7 @@ class NotificationMessageImpl implements NotificationMessage, Externalizable { private Set userIdSet = new HashSet() private String userGroupId = (String) null private String topic = (String) null + private String subTopic = (String) null private transient EntityValue notificationTopic = (EntityValue) null private String messageJson = (String) null private transient Map messageMap = (Map) null @@ -144,6 +145,9 @@ class NotificationMessageImpl implements NotificationMessage, Externalizable { @Override NotificationMessage topic(String topic) { this.topic = topic; notificationTopic = null; return this } @Override String getTopic() { topic } + @Override String getSubTopic() { subTopic } + @Override NotificationMessage subTopic(String st) { subTopic = st; return this } + @Override NotificationMessage message(String messageJson) { this.messageJson = messageJson; messageMap = null; return this } @Override NotificationMessage message(Map message) { this.messageMap = Collections.unmodifiableMap(message) as Map @@ -305,7 +309,7 @@ class NotificationMessageImpl implements NotificationMessage, Externalizable { boolean beganTransaction = tfi.begin(null) try { Map createResult = ecfi.service.sync().name("create", "moqui.security.user.NotificationMessage") - .parameters([topic:this.topic, userGroupId:this.userGroupId, sentDate:this.sentDate, + .parameters([topic:this.topic, subTopic:this.subTopic, userGroupId:this.userGroupId, sentDate:this.sentDate, messageJson:this.getMessageJson(), titleText:this.getTitle(), linkText:this.getLink(), typeString:this.getType(), showAlert:(this.showAlert ? 'Y' : 'N')]) .disableAuthz().call() @@ -450,7 +454,7 @@ class NotificationMessageImpl implements NotificationMessage, Externalizable { @Override Map getWrappedMessageMap() { EntityValue localNotTopic = getNotificationTopic() - return [topic:topic, sentDate:sentDate, notificationMessageId:notificationMessageId, topicDescription:localNotTopic?.description, + return [topic:topic, subTopic:subTopic, sentDate:sentDate, notificationMessageId:notificationMessageId, topicDescription:localNotTopic?.description, message:getMessageMap(), title:getTitle(), link:getLink(), type:getType(), persistOnSend:isPersistOnSend(), showAlert:isShowAlert(), alertNoAutoHide:isAlertNoAutoHide()] } @@ -467,6 +471,7 @@ class NotificationMessageImpl implements NotificationMessage, Externalizable { void populateFromValue(EntityValue nmbu) { this.notificationMessageId = nmbu.notificationMessageId this.topic = nmbu.topic + this.subTopic = nmbu.subTopic this.sentDate = nmbu.getTimestamp("sentDate") this.userGroupId = nmbu.userGroupId this.messageJson = nmbu.messageJson @@ -486,6 +491,7 @@ class NotificationMessageImpl implements NotificationMessage, Externalizable { out.writeObject(userIdSet) out.writeObject(userGroupId) out.writeUTF(topic) + out.writeObject(subTopic) out.writeUTF(getMessageJson()) out.writeObject(notificationMessageId) out.writeObject(sentDate) @@ -500,6 +506,7 @@ class NotificationMessageImpl implements NotificationMessage, Externalizable { userIdSet = (Set) objectInput.readObject() userGroupId = (String) objectInput.readObject() topic = objectInput.readUTF() + subTopic = objectInput.readObject() messageJson = objectInput.readUTF() notificationMessageId = (String) objectInput.readObject() sentDate = (Timestamp) objectInput.readObject() diff --git a/framework/src/main/java/org/moqui/context/NotificationMessage.java b/framework/src/main/java/org/moqui/context/NotificationMessage.java index b478e6121..38cb9636e 100644 --- a/framework/src/main/java/org/moqui/context/NotificationMessage.java +++ b/framework/src/main/java/org/moqui/context/NotificationMessage.java @@ -38,6 +38,9 @@ enum NotificationType { info, success, warning, danger } NotificationMessage topic(String topic); String getTopic(); + NotificationMessage subTopic(String subTopic); + String getSubTopic(); + /** Set the message as a JSON String. The top-level should be a Map (JSON Object). * @param messageJson The message as a JSON string containing a Map (JSON Object) * @return Self-reference for convenience From d42ae9c1fcc29f2e06744b2b5ca7cf8e0c6bb17b Mon Sep 17 00:00:00 2001 From: Acetousk Date: Mon, 13 Mar 2023 14:30:12 -0600 Subject: [PATCH 23/26] Refactor getTitle to prioritize title() method call over data --- .../context/NotificationMessageImpl.groovy | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy b/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy index 8c990c8c9..43988372d 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/NotificationMessageImpl.groovy @@ -173,16 +173,18 @@ class NotificationMessageImpl implements NotificationMessage, Externalizable { @Override NotificationMessage title(String title) { titleTemplate = title; return this } @Override String getTitle() { if (titleText == null) { - EntityValue localNotTopic = getNotificationTopic() - if (localNotTopic != null) { - if (type == danger && localNotTopic.errorTitleTemplate) { - titleText = ecfi.resource.expand((String) localNotTopic.errorTitleTemplate, "", getMessageMap(), true) - } else if (localNotTopic.titleTemplate) { - titleText = ecfi.resource.expand((String) localNotTopic.titleTemplate, "", getMessageMap(), true) + if (titleTemplate != null && !titleTemplate.isEmpty()) + titleText = ecfi.resource.expand(titleTemplate, "", getMessageMap(), true) + if (titleText == null || titleText.isEmpty()) { + EntityValue localNotTopic = getNotificationTopic() + if (localNotTopic != null) { + if (type == danger && localNotTopic.errorTitleTemplate) { + titleText = ecfi.resource.expand((String) localNotTopic.errorTitleTemplate, "", getMessageMap(), true) + } else if (localNotTopic.titleTemplate) { + titleText = ecfi.resource.expand((String) localNotTopic.titleTemplate, "", getMessageMap(), true) + } } } - if ((titleText == null || titleText.isEmpty()) && titleTemplate != null && !titleTemplate.isEmpty()) - titleText = ecfi.resource.expand(titleTemplate, "", getMessageMap(), true) } return titleText } From d85588600395b3f16400db33ccbe272ce351c6f2 Mon Sep 17 00:00:00 2001 From: Deepak Dixit Date: Tue, 11 Jul 2023 10:18:05 +0530 Subject: [PATCH 24/26] Upstream sync (#7) * Upgraded freemarker version to 2.3.32, also updated ftl version in FtlTemplateRenderer and MNode class (#565) https://freemarker.apache.org/docs/versions_2_3_32.html * Various library updates: jackson-databind, jetty, joda-time, shiro, sl4jf, junit * Added missing package name in entity relationship * Add StatusFlowTransitionFromAndTo view-entity * In ServiceFacadeImpl add classes for Service LoadRunner, used for load testing of service calls with some profiling info, some basics for a first pass * In Service LoadRunner add stats broken down by artifact type with detail, part of code is more generic addition to ArtifactExecutionInfoImpl * Service LoadRunner improved stats gathering with time bin based stats in addition to entire run stats, prep work for performance charts and such * In Service LoadRunner add small random delay support, use available threads * 4 for max load pool size; add permission for SERVICE_LOAD_RUNNER for screens * In build.gradle also delete the SaveOpenSearch.zip file in cleanLoadSave, also called by cleanAll, was missing in the prior OpenSearch changes * Fix PIT testing against ElasticSearch 7.10.2, the last open source version; the docs are horrible with 3 differences between what actually ended working from a bunch of random experimentation; PIT support is necessary for the Elastic Entity implementation (alternative to a DB cursor); this ended up being needed because more recent OpenSearch that supports PIT also uses more memory than ElasticSearch, enough that the anemic demo.moqui.org server with 2GB RAM can't run Moqui + OpenSearch 2.4 * Add new moqui-demo component repo to addons.xml * Add getScreenPathHasTransition in ScreenRenderImpl * In ElasticDatasourceFactory add better exceptions than NPE when no ElasticClient is found * Increase default worker-pool-core to 16 threads, and max to 32 or CPU * 3 threads (instead of *2); increasing after tests and production results showing typical low CPU time per thread (much of it is deferred database or search with lots of I/O wait); also improve logging when waiting for worker pool to empty; note that this also improves automated test performance because many of the tests get to hundreds of async services queued up and the ThreadPoolExecutor seems to add threads slowly even if the pool is fully utilized * In ScreenRenderImpl remove token created requirement when checking for session token * Revert "Update to session token condition" * Fixed a runtime error if Currency is BTC (#555) * Fixed the problem that moqui cannot be deployed as non-root webapp in Tomcat * Fixed a runtime error if Currency is BTC * Fixed a runtime error if Currency is BTC * Fixed the retries of Elastic Client * Anonymous usage of screens with transitions (#541) * fix: consider transitions in screens or parent screens defined with require-authentication in "anonymous-view" or "anonymous-all" as permitted * fix: consider transitions defined with authenticate in "anonymous-view" or "anonymous-all" as permitted * fix: consider any of service.@authenticate and screen.@require-authentication to determine permission on transition * fix: considering verb-based execution actions of single-service when determining transition permission, correctly determining whether view or all allowed by screen * Small fix for new authz changes for anonymous-view/-all, handle no screen definition instead of NPE * Handle usernames with different casing in session data * Add common java includes to the xml actions ftl file * Add subTopic field to NotificationMessage * Refactor getTitle to prioritize title() method call over data * In MoquiStart add support for webapp_session_cookie_max_age env var to set the cookie expire time based on this age in seconds, if not specified defaults to Session expire and cookie will be dropped by the browser when it quits * BugFix for DataFeed could not find backward relationship for DataDocument * Add new WeCreate application component to addons.xml * Allow for properties to be marked "is-secret" so their values don't get printed into the log at startup. * Add weight microgram weight unit of measure. * Small change to new default-property.@is-secret attribute so that a false value treats it as non-secret even if it contains one of the previously supported substrings (pass, pw, key); add explicit is-secret=true attrs to a few in MoquiDefaultConf.xml * In ScreenRenderImpl add methods needed for section-include to support pagination; in CollectionUtilities paginateList() and paginateParameters() add support for entity-find with search-form-inputs or other situations where paginate parameters are already in place, don't overwrite them and don't try to subList() a big list * Add new moqui-image repo, fork of original work be acetousk * Add Sales app from xolvegroup (third party component) * Added support for loading data on start with entity_on_start_load_types and entity_on_start_load_components properties/env-vars --------- Co-authored-by: David E Jones Co-authored-by: Acetousk Co-authored-by: aabiabdallah Co-authored-by: Wei Zhang Co-authored-by: Jens Hardings Co-authored-by: Yao Chunlin Co-authored-by: Adam Heath --- addons.xml | 3 + framework/data/UnitData.xml | 2 + .../ExecutionContextFactoryImpl.groovy | 130 ++++++++----- .../moqui/impl/entity/EntityDataFeed.groovy | 2 +- .../moqui/impl/screen/ScreenRenderImpl.groovy | 17 +- .../context/ExecutionContextFactory.java | 4 +- .../org/moqui/util/CollectionUtilities.java | 36 ++-- .../src/main/resources/MoquiDefaultConf.xml | 178 +++++++++--------- framework/src/start/java/MoquiStart.java | 14 ++ framework/xsd/moqui-conf-3.xsd | 9 + 10 files changed, 245 insertions(+), 150 deletions(-) diff --git a/addons.xml b/addons.xml index 8e7e0e3de..d449211e1 100644 --- a/addons.xml +++ b/addons.xml @@ -47,6 +47,7 @@ + @@ -77,6 +78,7 @@ + @@ -92,6 +94,7 @@ + @@ -67,7 +69,7 @@ - + @@ -75,13 +77,15 @@ - + - + @@ -97,23 +101,23 @@ + key-type="org.moqui.entity.EntityCondition" value-type="org.moqui.impl.entity.EntityValueBase"/> + key-type="org.moqui.entity.EntityCondition" value-type="org.moqui.impl.entity.EntityListImpl"/> + key-type="org.moqui.entity.EntityCondition" value-type="Long"/> + key-type="org.moqui.entity.EntityCondition" value-type="Set"/> + key-type="org.moqui.entity.EntityCondition" value-type="Set"/> + value-type="Set"/> + key-type="org.moqui.entity.EntityCondition" value-type="Set"/> + key-type="org.moqui.entity.EntityCondition" value-type="Set"/> @@ -267,12 +271,12 @@ + value="Access-Control-Allow-Origin,Access-Control-Allow-Credentials,X-CSRF-Token,moquiSessionToken"/> + value="Accept,Accept-Encoding,Content-Type,Content-Encoding,Origin,X-Requested-With,Access-Control-Request-Method,Access-Control-Request-Headers,api_key,login_key,X-HTTP-Method-Override,moquiSessionToken,SessionToken,X-CSRF-Token,Authorization"/> @@ -354,35 +358,35 @@ + macro-template-location="template/screen-macro/DefaultScreenMacros.csv.ftl"/> + macro-template-location="template/screen-macro/DefaultScreenMacros.html.ftl"/> + macro-template-location="template/screen-macro/DefaultScreenMacros.text.ftl"/> + macro-template-location="template/screen-macro/DefaultScreenMacros.xml.ftl"/> + macro-template-location="template/screen-macro/DefaultScreenMacros.xsl-fo.ftl"/> + macro-template-location="template/screen-macro/DefaultScreenMacros.vuet.ftl"/> + macro-template-location="template/screen-macro/DefaultScreenMacros.plain.ftl"/> + macro-template-location="template/screen-macro/DefaultScreenMacros.plain.ftl"/> + macro-template-location="template/screen-macro/DefaultScreenMacros.qvt.ftl"/> + macro-template-location="template/screen-macro/DefaultScreenMacros.plain.ftl"/> + macro-template-location="template/screen-macro/DefaultScreenMacros.plain.ftl"/> + job-queue-max="0" job-pool-core="2" job-pool-max="8" job-pool-alive="120"> @@ -417,13 +421,13 @@ + index-prefix="${elasticsearch_index_prefix}" pool-max="64" queue-size="1024"/> + distributed-cache-invalidate="false" dci-topic-factory="" query-stats="false" + database-locale="${default_locale}" database-time-zone="${database_time_zone ?: default_time_zone}" + crypt-salt="20201202" crypt-iter="10" crypt-algo="PBEWithHmacSHA256AndAES_128" crypt-pass="${entity_ds_crypt_pass}"> @@ -462,7 +466,7 @@ few extra so 120 connections in the main pool. --> + runtime-add-missing="${entity_add_missing_runtime}" startup-add-missing="${entity_add_missing_startup}"> @@ -479,18 +483,18 @@ This will also work for Postgres but you will get warnings about errors setting invalid properties on the XADataSource. --> + runtime-add-missing="false" startup-add-missing="false" disabled="${entity_ds_c1_disabled}"> + databaseName="${entity_ds_c1_database}" user="${entity_ds_c1_user}" password="${entity_ds_c1_password}" + pinGlobalTxToPhysicalConnection="true" autoReconnectForPools="true" useUnicode="true" encoding="UTF-8"/> + object-factory="org.moqui.impl.entity.elastic.ElasticDatasourceFactory"> @@ -574,12 +578,12 @@ --> + default-isolation-level="ReadCommitted" for-update="FOR UPDATE WITH RS" + use-schema-for-all="true" use-indexes-unique="false" use-pk-constraint-names="false" fk-style="name_fk" + default-test-query="SELECT 1 FROM SYSIBM.SYSDUMMY1" + default-jdbc-driver="com.ibm.db2.jcc.DB2Driver" + default-xa-ds-class="com.ibm.db2.jcc.DB2XADataSource" + default-startup-add-missing="true" default-runtime-add-missing="false"> @@ -587,15 +591,15 @@ + databaseName="${entity_ds_database}" user="${entity_ds_user}" password="${entity_ds_password}"/> + default-isolation-level="ReadCommitted" for-update="FOR UPDATE WITH RS" + use-schema-for-all="true" use-indexes-unique-where-not-null="true" + default-test-query="SELECT 1 FROM SYSIBM.SYSDUMMY1" + default-jdbc-driver="com.ibm.as400.access.AS400JDBCDriver" + default-xa-ds-class="com.ibm.as400.access.AS400JDBCXADataSource" + default-startup-add-missing="true" default-runtime-add-missing="false"> @@ -603,7 +607,7 @@ + user="${entity_ds_user}" password="${entity_ds_password}" cursorHold="false" threadUsed="false"/> + default-jdbc-driver="org.apache.derby.jdbc.EmbeddedDriver" + default-xa-ds-class="org.apache.derby.jdbc.EmbeddedXADataSource"> @@ -632,16 +636,16 @@ --> + default-jdbc-driver="org.h2.Driver" default-xa-ds-class="org.h2.jdbcx.JdbcDataSource" + default-start-server-args="-tcpPort 9092 -ifExists -baseDir ${moqui_runtime}/db/h2"> + default-isolation-level="ReadUncommitted" default-jdbc-driver="org.hsqldb.jdbcDriver" + default-test-query="SELECT 1 FROM SEQUENCE_VALUE_ITEM WHERE 1=0"> @@ -674,9 +678,9 @@ NOTE: for MS SQL Server 2012 and later can use offset-style=fetch for better performance and consistent behavior --> + default-test-query="SELECT 1" default-jdbc-driver="com.microsoft.sqlserver.jdbc.SQLServerDriver" + default-xa-ds-class="com.microsoft.sqlserver.jdbc.SQLServerXADataSource" + default-startup-add-missing="true" default-runtime-add-missing="false" never-nulls="true"> @@ -701,8 +705,8 @@ + databaseName="${entity_ds_database}" user="${entity_ds_user}" password="${entity_ds_password}" + selectMethod="Cursor"/> + table-engine="InnoDB" character-set="utf8" collate="utf8_general_ci" fk-style="name_fk" + constraint-name-clip-length="60" + default-isolation-level="ReadCommitted" default-test-query="SELECT 1" + default-jdbc-driver="com.mysql.jdbc.Driver" + default-xa-ds-class="com.mysql.jdbc.jdbc2.optional.MysqlXADataSource"> + pinGlobalTxToPhysicalConnection="true" autoReconnectForPools="true" useUnicode="true" encoding="UTF-8" useCursorFetch="true" + databaseName="${entity_ds_database}" user="${entity_ds_user}" password="${entity_ds_password}"/> + never-nulls="true" table-engine="InnoDB" character-set="utf8" collate="utf8_general_ci" fk-style="name_fk" + constraint-name-clip-length="60" + default-isolation-level="ReadCommitted" default-test-query="SELECT 1" + default-startup-add-missing="true" default-runtime-add-missing="false" + default-jdbc-driver="com.mysql.cj.jdbc.Driver" + default-xa-ds-class="com.mysql.cj.jdbc.MysqlXADataSource"> + pinGlobalTxToPhysicalConnection="true" autoReconnectForPools="true" useCursorFetch="true" + serverTimezone="${database_time_zone ?: default_time_zone}" useSSL="false" allowPublicKeyRetrieval="true" + databaseName="${entity_ds_database}" user="${entity_ds_user}" password="${entity_ds_password}"/> + default-test-query="SELECT 1 FROM DUAL" default-jdbc-driver="oracle.jdbc.driver.OracleDriver" + default-xa-ds-class="oracle.jdbc.xa.client.OracleXADataSource" + default-startup-add-missing="true" default-runtime-add-missing="false"> @@ -818,7 +822,7 @@ + URL="jdbc:oracle:thin:@${entity_ds_host}:${entity_ds_port?:'1521'}:${entity_ds_database}"/> + never-try-insert="true" default-isolation-level="ReadCommitted" use-tm-join="true" default-test-query="SELECT 1" + constraint-name-clip-length="60" + default-jdbc-driver="org.postgresql.Driver" default-xa-ds-class="org.postgresql.xa.PGXADataSource" + default-startup-add-missing="true" default-runtime-add-missing="false" use-binary-type-for-blob="true"> @@ -854,7 +858,7 @@ + databaseName="${entity_ds_database}" user="${entity_ds_user}" password="${entity_ds_password}"/> diff --git a/framework/src/start/java/MoquiStart.java b/framework/src/start/java/MoquiStart.java index 031366779..73010a320 100644 --- a/framework/src/start/java/MoquiStart.java +++ b/framework/src/start/java/MoquiStart.java @@ -282,6 +282,20 @@ public static void main(String[] args) throws IOException { // NOTE DEJ20210520: now always using StartClassLoader because of breaking classloader changes in 9.4.37 (likely from https://github.com/eclipse/jetty.project/pull/5894) webappClass.getMethod("setClassLoader", ClassLoader.class).invoke(webapp, moquiStartLoader); + // handle webapp_session_cookie_max_age with setInitParameter (1209600 seconds is about 2 weeks 60 * 60 * 24 * 14) + String sessionMaxAge = System.getenv("webapp_session_cookie_max_age"); + if (sessionMaxAge != null && !sessionMaxAge.isEmpty()) { + Integer maxAgeInt = null; + try { maxAgeInt = Integer.parseInt(sessionMaxAge); } + catch (Exception e) { System.out.println("Found webapp_session_cookie_max_age env var with invalid number, ignoring: " + sessionMaxAge); } + + if (maxAgeInt != null) { + System.out.println("Setting Servlet Session Max Age based on webapp_session_cookie_max_age " + maxAgeInt); + webappClass.getMethod("setInitParameter", String.class, String.class) + .invoke(webapp, "org.eclipse.jetty.servlet.MaxAge", maxAgeInt.toString()); + } + } + // WebSocket Object wsContainer = wsInitializerClass.getMethod("configure", scHandlerClass, wsInitializerConfiguratorClass).invoke(null, webapp, null); webappClass.getMethod("setAttribute", String.class, Object.class).invoke(webapp, "javax.websocket.server.ServerContainer", wsContainer); diff --git a/framework/xsd/moqui-conf-3.xsd b/framework/xsd/moqui-conf-3.xsd index fa88f3b45..d060a6176 100644 --- a/framework/xsd/moqui-conf-3.xsd +++ b/framework/xsd/moqui-conf-3.xsd @@ -49,6 +49,7 @@ along with this software (see the LICENSE.md file). If not, see + @@ -60,6 +61,14 @@ along with this software (see the LICENSE.md file). If not, see table for moqui.basic.Enumeration). Empty or 'none' means load nothing, use 'all' to load all found data files regardless of type. + + Comma-separated list of data file types to load on start. Empty or 'none' means load nothing. + Does not run if empty-db-load runs. + + + Comma-separated list of component names to load on start, used with on-start-load-types. + Does not run if empty-db-load runs. + The maximum size of the worker queue. From dd5985a9894af00efb7c6f25eb9b7a9e1875ae3a Mon Sep 17 00:00:00 2001 From: Deepak Dixit Date: Mon, 6 Nov 2023 14:44:43 +0530 Subject: [PATCH 25/26] Master (#9) * Fix regression with partitioned tables in PostgreSQL PostgreSQL JDBC Driver introduced separating type for partitioned table from 40.2.12 pgjdbc/pgjdbc#1708 * Add subscreensItem.menuInclude to menu data (#600) * Fixed the problem that moqui cannot be deployed as non-root webapp in Tomcat * Fixed a runtime error if Currency is BTC * Fixed a runtime error if Currency is BTC * Fixed the retries of Elastic Client * Add subscreensItem.menuInclude to menu data * Docker-Image-Pull Feature (#553) * Improvement: In Moqui-Multi-Instance added a Async-Pull-Image-Feature which pulls the image by Using Docker-Engine-Api from multiple registry like AWS, Azure, Docker-Hub * Update AUTHORS * Added: added the generic way to process cmd , by adding an extra field(authTokenPass) inside the entity InstanceImage , now user has separate field for cmd and password so all types of registries is easily configurable * Update ServerEntities.xml by adding description * Library updates, including Jetty 10.0.13 to 10.015 which had reported vulnerabilities; there are lots of dependencies updated in this set, see diff for full details * In ScreenRenderImpl change addFormFieldValue() and related methods to handle first, second, and last rows, for qvt and other client rendered output that needs full data for a form in a map/object * In root build.gradle change gitStatusAll task to be more tolerant of repos with no master branch * Add text-area.@autogrow attribute, supported only in qvt for now * Update various libraries including Groovy to 3.0.19 (which has some minor non-backward compatible changes single 3.0.10 with odd boolean behavior in rare cases, adjusted for in the framework long ago but should be watched for in custom code), Jetty to 10.0.16, H2 database, SLF4J, SnakeYAML, Apache Commons Lang3 * In build.gradle gitStatusAll task also handle upstream remotes with no master branch * Currency (#614) * Use moqui.basic.Uom entity to determine currency formatting and rounding details * Add currency-hide-symbol attribute as a complement to currency-unit-field, displaying the value without the currency symbol * Update authors file * Add and Handle Hmac Sha256 with timestamp * A couple of minor bug fixes in the EntityAutoServiceRunner and ContextJavaUtil (#618) * In EntityAutoServiceRunner, remove unwanted break statement to ensure support for multiple PK fields with wildcard (*). * In ContextJavaUtil, add missing future keyword. * Updated authors file. * Allow for 10 second threshold in nowTimestamp * In L10nFacadeImpl.formatCurrency() use disableAuthz() for entity find on Uom; small change to currency formatting test to pass with current OOTB settings * In addons.xml, added new moqui-sso component. --------- Co-authored-by: Yao Chunlin Co-authored-by: David E. Jones Co-authored-by: Wei Zhang Co-authored-by: Rohit pawar <72196393+rohitpawar2811@users.noreply.github.com> Co-authored-by: Jens Hardings Co-authored-by: acetousk Co-authored-by: Ayman Abi Abdallah --- AUTHORS | 10 +- addons.xml | 1 + build.gradle | 25 +++-- framework/build.gradle | 84 ++++++++-------- framework/entity/BasicEntities.xml | 4 +- framework/entity/ServerEntities.xml | 3 + framework/entity/ServiceEntities.xml | 1 + .../org/moqui/impl/InstanceServices.xml | 60 +++++++++++- .../moqui/impl/context/ContextJavaUtil.java | 3 +- .../impl/context/ElasticFacadeImpl.groovy | 2 +- .../moqui/impl/context/L10nFacadeImpl.java | 97 ++++++++++++++----- .../moqui/impl/context/WebFacadeImpl.groovy | 66 +++++++++++++ .../org/moqui/impl/entity/EntityDbMeta.groovy | 4 +- .../moqui/impl/screen/ScreenRenderImpl.groovy | 52 +++++++--- .../runner/EntityAutoServiceRunner.groovy | 1 - .../java/org/moqui/context/L10nFacade.java | 7 +- .../src/test/groovy/L10nFacadeTests.groovy | 2 +- framework/xsd/xml-form-3.xsd | 8 ++ 18 files changed, 328 insertions(+), 102 deletions(-) diff --git a/AUTHORS b/AUTHORS index 27dbd1a34..28e89ca3e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -46,7 +46,7 @@ Written in 2015 by Sam Hamilton - samhamilton Written in 2015 by Leonardo Carvalho - CarvalhoLeonardo Written in 2015 by Swapnil M Mane - swapnilmmane Written in 2015 by Anton Akhiar - akhiar -Written in 2015-2018 by Jens Hardings - jenshp +Written in 2015-2023 by Jens Hardings - jenshp Written in 2016 by Shifeng Zhang - zhangshifeng Written in 2016 by Scott Gray - lektran Written in 2016 by Mark Haney - mphaney @@ -54,13 +54,14 @@ Written in 2016 by Qiushi Yan - yanqiushi Written in 2017 by Oleg Andrieiev - oandreyev Written in 2018 by Zhang Wei - zhangwei1979 Written in 2018 by Nirendra Singh - nirendra10695 -Written in 2018-2021 by Ayman Abi Abdallah - aabiabdallah +Written in 2018-2023 by Ayman Abi Abdallah - aabiabdallah Written in 2019 by Daniel Taylor - danieltaylor-nz Written in 2020 by Jacob Barnes - Tellan Written in 2020 by Amir Anjomshoaa - amiranjom Written in 2021 by Deepak Dixit - dixitdeepak Written in 2021 by Taher Alkhateeb - pythys Written in 2022 by Zhang Wei - hellozhangwei +Written in 2023 by Rohit Pawar - rohitpawar2811 =========================================================================== @@ -93,7 +94,7 @@ Written in 2015 by Jimmy Shen - shendepu Written in 2015-2016 by Sam Hamilton - samhamilton Written in 2015 by Leonardo Carvalho - CarvalhoLeonardo Written in 2015 by Anton Akhiar - akhiar -Written in 2015-2016 by Jens Hardings - jenshp +Written in 2015-2023 by Jens Hardings - jenshp Written in 2016 by Shifeng Zhang - zhangshifeng Written in 2016 by Scott Gray - lektran Written in 2016 by Mark Haney - mphaney @@ -101,12 +102,13 @@ Written in 2016 by Qiushi Yan - yanqiushi Written in 2017 by Oleg Andrieiev - oandreyev Written in 2018 by Zhang Wei - zhangwei1979 Written in 2018 by Nirendra Singh - nirendra10695 -Written in 2018-2020 by Ayman Abi Abdallah - aabiabdallah +Written in 2018-2023 by Ayman Abi Abdallah - aabiabdallah Written in 2019 by Daniel Taylor - danieltaylor-nz Written in 2020 by Jacob Barnes - Tellan Written in 2020 by Amir Anjomshoaa - amiranjom Written in 2021 by Deepak Dixit - dixitdeepak Written in 2021 by Taher Alkhateeb - pythys Written in 2022 by Zhang Wei - hellozhangwei +Written in 2023 by Rohit Pawar - rohitpawar2811 =========================================================================== diff --git a/addons.xml b/addons.xml index d449211e1..f5d4f2aac 100644 --- a/addons.xml +++ b/addons.xml @@ -52,6 +52,7 @@ + diff --git a/build.gradle b/build.gradle index 3b591d4c6..3fc65c0e6 100644 --- a/build.gradle +++ b/build.gradle @@ -417,13 +417,26 @@ task gitStatusAll { def curGrgit = Grgit.open(dir: gitDir) logger.lifecycle("\nGit status for ${gitDir} (branch:${curGrgit.branch.current()?.name}, tracking:${curGrgit.branch.current()?.trackingBranch?.name})") - if (curGrgit.remote.list().find({ it.name == 'upstream'})) { - def upstreamAhead = curGrgit.log { range curGrgit.resolve.toCommit('refs/remotes/upstream/master'), curGrgit.resolve.toCommit('refs/remotes/origin/master') } - if (upstreamAhead) logger.lifecycle("- origin/master ${upstreamAhead.size()} commits ahead of upstream/master") + try { + if (curGrgit.remote.list().find({ it.name == 'upstream'})) { + def upstreamAhead = curGrgit.log { range curGrgit.resolve.toCommit('refs/remotes/upstream/master'), curGrgit.resolve.toCommit('refs/remotes/origin/master') } + if (upstreamAhead) logger.lifecycle("- origin/master ${upstreamAhead.size()} commits ahead of upstream/master") + } + } catch (Exception e) { + logger.error("Error finding commits ahead of upstream", e) + } + try { + def masterLatest = curGrgit.resolve.toCommit('refs/remotes/origin/master') + if (masterLatest == null) { + logger.error("No origin/master branch exists, can't determine unpushed commits") + } else { + def unpushed = curGrgit.log { range masterLatest, curGrgit.resolve.toCommit('HEAD') } + if (unpushed) logger.lifecycle("--- ${unpushed.size()} commits unpushed (ahead of origin/master)") + for (Commit commit in unpushed) logger.lifecycle(" - ${commit.getAbbreviatedId(8)} - ${commit.shortMessage}") + } + } catch (Exception e) { + logger.error("Error finding unpushed commits", e) } - def unpushed = curGrgit.log { range curGrgit.resolve.toCommit('refs/remotes/origin/master'), curGrgit.resolve.toCommit('HEAD') } - if (unpushed) logger.lifecycle("--- ${unpushed.size()} commits unpushed (ahead of origin/master)") - for (Commit commit in unpushed) logger.lifecycle(" - ${commit.getAbbreviatedId(8)} - ${commit.shortMessage}") def curStatus = curGrgit.status() if (curStatus.isClean()) logger.lifecycle("* nothing to commit, working directory clean") if (curStatus.staged.added || curStatus.staged.modified || curStatus.staged.removed) logger.lifecycle("--- Changes to be committed::") diff --git a/framework/build.gradle b/framework/build.gradle index 8bc029de6..65137c15c 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -12,7 +12,7 @@ * . */ -version = '3.1.0-rc1' +version = '3.1.0-rc2' apply plugin: 'java-library' apply plugin: 'groovy' @@ -27,7 +27,7 @@ buildscript { maven { url "https://plugins.gradle.org/m2/" } } dependencies { - classpath 'com.github.ben-manes:gradle-versions-plugin:0.45.0' + classpath 'com.github.ben-manes:gradle-versions-plugin:0.47.0' // uncomment to add the Error Prone compiler: classpath 'net.ltgt.gradle:gradle-errorprone-plugin:0.0.8' } } @@ -68,17 +68,17 @@ tasks.withType(GroovyCompile) { options.compilerArgs << "-proc:none" } // NOTE: for dependency types and 'api' definition see: https://docs.gradle.org/current/userguide/java_library_plugin.html dependencies { // Groovy - // NOTE: Groovy 3.0.10-3.0.12 has a bug that somehow causes EntityDefinition.isViewEntity (public final boolean) to switch + // NOTE: Groovy 3.0.10-3.0.18 has a bug that somehow causes EntityDefinition.isViewEntity (public final boolean) to switch // from true during constructor to false later on; see EntityFindBuilder.java:112-114 and EntityDefinition.groovy:50-53,94-95; // for now using Boolean instead of boolean to resolve, but staying at 3.0.9 to avoid risk with other code - api 'org.codehaus.groovy:groovy:3.0.9' // Apache 2.0 - api 'org.codehaus.groovy:groovy-dateutil:3.0.9' // Apache 2.0 - api 'org.codehaus.groovy:groovy-groovysh:3.0.9' // Apache 2.0 + api 'org.codehaus.groovy:groovy:3.0.19' // Apache 2.0 + api 'org.codehaus.groovy:groovy-dateutil:3.0.19' // Apache 2.0 + api 'org.codehaus.groovy:groovy-groovysh:3.0.19' // Apache 2.0 // jline, an older version, is required by groovy-groovysh but not in its dependencies implementation 'jline:jline:2.14.6' // BSD - api 'org.codehaus.groovy:groovy-json:3.0.9' // Apache 2.0 - api 'org.codehaus.groovy:groovy-templates:3.0.9' // Apache 2.0 - api 'org.codehaus.groovy:groovy-xml:3.0.9' // Apache 2.0 + api 'org.codehaus.groovy:groovy-json:3.0.19' // Apache 2.0 + api 'org.codehaus.groovy:groovy-templates:3.0.19' // Apache 2.0 + api 'org.codehaus.groovy:groovy-xml:3.0.19' // Apache 2.0 // jansi is needed for groovydoc only, so in compileOnly (not included in war) - don't update to version 2, keep at version 1 for compatibility compileOnly 'org.fusesource.jansi:jansi:1.18' // Findbugs need only during compile (used by freemarker and various moqui classes) @@ -93,33 +93,33 @@ dependencies { // ========== General Libraries from Maven Central ========== // Apache Commons - api 'org.apache.commons:commons-csv:1.9.0' // Apache 2.0 + api 'org.apache.commons:commons-csv:1.10.0' // Apache 2.0 // NOTE: commons-email depends on com.sun.mail:javax.mail, included below, so use module() here to not get dependencies api module('org.apache.commons:commons-email:1.5') // Apache 2.0 - api 'org.apache.commons:commons-lang3:3.12.0' // Apache 2.0; used by cron-utils + api 'org.apache.commons:commons-lang3:3.13.0' // Apache 2.0; used by cron-utils api 'commons-beanutils:commons-beanutils:1.9.4' // Apache 2.0 - api 'commons-codec:commons-codec:1.15' // Apache 2.0 + api 'commons-codec:commons-codec:1.16.0' // Apache 2.0 api 'commons-collections:commons-collections:3.2.2' // Apache 2.0 api 'commons-digester:commons-digester:2.1' // Apache 2.0 - api 'commons-fileupload:commons-fileupload:1.4' // Apache 2.0 - api 'commons-io:commons-io:2.11.0' // Apache 2.0 + api 'commons-fileupload:commons-fileupload:1.5' // Apache 2.0 + api 'commons-io:commons-io:2.13.0' // Apache 2.0 api 'commons-logging:commons-logging:1.2' // Apache 2.0 api 'commons-validator:commons-validator:1.7' // Apache 2.0 // Cron Utils - api 'com.cronutils:cron-utils:9.2.0' // Apache 2.0 + api 'com.cronutils:cron-utils:9.2.1' // Apache 2.0 // Flexmark (markdown) - api 'com.vladsch.flexmark:flexmark:0.64.0' - api 'com.vladsch.flexmark:flexmark-ext-tables:0.64.0' - api 'com.vladsch.flexmark:flexmark-ext-toc:0.64.0' + api 'com.vladsch.flexmark:flexmark:0.64.8' + api 'com.vladsch.flexmark:flexmark-ext-tables:0.64.8' + api 'com.vladsch.flexmark:flexmark-ext-toc:0.64.8' // Freemarker // Remember to change the version number in FtlTemplateRenderer and MNode class when upgrading api 'org.freemarker:freemarker:2.3.32' // Apache 2.0 // H2 Database - api 'com.h2database:h2:2.1.214' // MPL 2.0, EPL 1.0 + api 'com.h2database:h2:2.2.222' // MPL 2.0, EPL 1.0 // Java Specifications api 'javax.transaction:jta:1.1' @@ -144,38 +144,38 @@ dependencies { api 'com.beust:jcommander:1.82' // Jackson Databind (JSON, etc) - api 'com.fasterxml.jackson.core:jackson-databind:2.14.2' + api 'com.fasterxml.jackson.core:jackson-databind:2.15.2' // Jetty HTTP Client and Proxy Servlet - api 'org.eclipse.jetty:jetty-client:10.0.13' // Apache 2.0 - api 'org.eclipse.jetty:jetty-proxy:10.0.13' // Apache 2.0 + api 'org.eclipse.jetty:jetty-client:10.0.16' // Apache 2.0 + api 'org.eclipse.jetty:jetty-proxy:10.0.16' // Apache 2.0 // javax.mail // NOTE: javax.mail depends on 'javax.activation:activation' which is the old package for 'javax.activation:javax.activation-api' used by jaxb-api api module('com.sun.mail:javax.mail:1.6.2') // CDDL // Joda Time (used by elasticsearch, aws) - api 'joda-time:joda-time:2.12.2' // Apache 2.0 + api 'joda-time:joda-time:2.12.5' // Apache 2.0 // JSoup (HTML parser, cleaner) - api 'org.jsoup:jsoup:1.15.3' // MIT + api 'org.jsoup:jsoup:1.16.1' // MIT // Apache Shiro - api module('org.apache.shiro:shiro-core:1.11.0') // Apache 2.0 - api module('org.apache.shiro:shiro-web:1.11.0') // Apache 2.0 + api module('org.apache.shiro:shiro-core:1.12.0') // Apache 2.0 + api module('org.apache.shiro:shiro-web:1.12.0') // Apache 2.0 // SLF4J, Log4j 2 (note Log4j 2 is used by various libraries, best not to replace it even if mostly possible with SLF4J) - api 'org.slf4j:slf4j-api:2.0.6' - implementation 'org.apache.logging.log4j:log4j-core:2.19.0' - implementation 'org.apache.logging.log4j:log4j-api:2.19.0' - runtimeOnly 'org.apache.logging.log4j:log4j-jcl:2.19.0' - runtimeOnly 'org.apache.logging.log4j:log4j-slf4j2-impl:2.19.0' + api 'org.slf4j:slf4j-api:2.0.9' + implementation 'org.apache.logging.log4j:log4j-core:2.20.0' + implementation 'org.apache.logging.log4j:log4j-api:2.20.0' + runtimeOnly 'org.apache.logging.log4j:log4j-jcl:2.20.0' + runtimeOnly 'org.apache.logging.log4j:log4j-slf4j2-impl:2.20.0' // SubEtha SMTP (module as depends on old javax.mail location; also uses SLF4J, activation included elsewhere) api module('org.subethamail:subethasmtp:3.1.7') // Snake YAML - api 'org.yaml:snakeyaml:1.33' // Apache 2.0 + api 'org.yaml:snakeyaml:2.2' // Apache 2.0 // Apache Jackrabbit - uncomment here or include elsewhere when Jackrabbit repository configurations are used // api 'org.apache.jackrabbit:jackrabbit-jcr-rmi:2.12.1' // Apache 2.0 @@ -190,11 +190,11 @@ dependencies { // ========== test dependencies ========== // junit-platform-launcher is a dependency from spock-core, included explicitly to get more recent version as needed - testImplementation 'org.junit.platform:junit-platform-launcher:1.9.2' + testImplementation 'org.junit.platform:junit-platform-launcher:1.10.0' // junit-platform-suite required for test suites to specify test class order, etc - testImplementation 'org.junit.platform:junit-platform-suite:1.9.2' + testImplementation 'org.junit.platform:junit-platform-suite:1.10.0' // junit-jupiter-api for using JUnit directly, not generally needed for Spock based tests - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.2' + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.0' // Spock Framework testImplementation platform("org.spockframework:spock-bom:2.1-groovy-3.0") // Apache 2.0 testImplementation 'org.spockframework:spock-core:2.1-groovy-3.0' // Apache 2.0 @@ -203,16 +203,16 @@ dependencies { // ========== executable war dependencies ========== // Jetty - execWarRuntimeOnly 'org.eclipse.jetty:jetty-server:10.0.13' // Apache 2.0 - execWarRuntimeOnly 'org.eclipse.jetty:jetty-webapp:10.0.13' // Apache 2.0 - execWarRuntimeOnly 'org.eclipse.jetty:jetty-jndi:10.0.13' // Apache 2.0 - execWarRuntimeOnly 'org.eclipse.jetty.websocket:websocket-javax-server:10.0.13' // Apache 2.0 - execWarRuntimeOnly ('org.eclipse.jetty.websocket:websocket-javax-client:10.0.13') { // Apache 2.0 + execWarRuntimeOnly 'org.eclipse.jetty:jetty-server:10.0.16' // Apache 2.0 + execWarRuntimeOnly 'org.eclipse.jetty:jetty-webapp:10.0.16' // Apache 2.0 + execWarRuntimeOnly 'org.eclipse.jetty:jetty-jndi:10.0.16' // Apache 2.0 + execWarRuntimeOnly 'org.eclipse.jetty.websocket:websocket-javax-server:10.0.16' // Apache 2.0 + execWarRuntimeOnly ('org.eclipse.jetty.websocket:websocket-javax-client:10.0.16') { // Apache 2.0 exclude group: 'javax.websocket' } // we have the full websocket API, including the client one causes problems execWarRuntimeOnly 'javax.websocket:javax.websocket-api:1.1' - execWarRuntimeOnly ('org.eclipse.jetty.websocket:websocket-jetty-server:10.0.13') // Apache 2.0 + execWarRuntimeOnly ('org.eclipse.jetty.websocket:websocket-jetty-server:10.0.16') // Apache 2.0 // only include this if using Endpoint and MessageHandler annotations: - // execWarRuntime ('org.eclipse.jetty:jetty-annotations:10.0.13') // Apache 2.0 + // execWarRuntime ('org.eclipse.jetty:jetty-annotations:10.0.16') // Apache 2.0 execWarRuntimeOnly 'org.apache.logging.log4j:log4j-slf4j18-impl:2.18.0' } diff --git a/framework/entity/BasicEntities.xml b/framework/entity/BasicEntities.xml index 3ca7a10e4..d71b77fc5 100644 --- a/framework/entity/BasicEntities.xml +++ b/framework/entity/BasicEntities.xml @@ -389,7 +389,9 @@ along with this software (see the LICENSE.md file). If not, see - + + + diff --git a/framework/entity/ServerEntities.xml b/framework/entity/ServerEntities.xml index 17ec6556d..e9b63af25 100644 --- a/framework/entity/ServerEntities.xml +++ b/framework/entity/ServerEntities.xml @@ -196,6 +196,9 @@ along with this software (see the LICENSE.md file). If not, see + + + diff --git a/framework/entity/ServiceEntities.xml b/framework/entity/ServiceEntities.xml index 81696e90f..7b51a3ab3 100644 --- a/framework/entity/ServiceEntities.xml +++ b/framework/entity/ServiceEntities.xml @@ -383,6 +383,7 @@ along with this software (see the LICENSE.md file). If not, see + diff --git a/framework/service/org/moqui/impl/InstanceServices.xml b/framework/service/org/moqui/impl/InstanceServices.xml index 7071d77fa..dacbd18ca 100644 --- a/framework/service/org/moqui/impl/InstanceServices.xml +++ b/framework/service/org/moqui/impl/InstanceServices.xml @@ -214,8 +214,13 @@ along with this software (see the LICENSE.md file). If not, see - - + + + @@ -281,8 +286,9 @@ along with this software (see the LICENSE.md file). If not, see - https://docs.docker.com/engine/security/https/ --> - + +