Skip to content

Commit

Permalink
Merge 21.05 into master (#20425)
Browse files Browse the repository at this point in the history
* Upgrading webcomponents version to 0.0.25

* Issue 20329 osgi not working java 8 (#20360)

* Switch byte-buddy version depending on java used to run

* Remove echo

* Issue 19646 improvment memory space consumption (#20370)

* #19646 Improvment heap scape consumption generating a bundle

* #19646 Testing

* Change ee version in dependencies.properties

* Revert DotSubmitterchange and validate empty file

* Rever dependencies.properties change

* run-all

* Bump up mysql connector version to be able to run in Java 11 (#20373)

* Bump up mysql connector version to be able to run in Java 11

* run-all

* #20358 earlier releases had a metadata column which isn't used anymore… (#20372)

* #20358 earlier releases had a metadata column which isnt used anymore and it needs to be removed.

* #20358 doc test

* run-all

* run-all

* adding doc

* run-all

* #20336 Improving task performance (#20374)

* #20336 Improving task performance

* run-all

* run-all

* Removing semicolon in oracle script

* run-all

* run-all

* Adding missing semicolon

* run-all

* run-oracle

Co-authored-by: Nollymar Longa <>

* #20312 Upserts (#20367)

* #20312 Upserts

* run-all

* #20312 sql merge clean up

* #20312 fixing  typos

* run-all

* run-all

* run-all

* #20312 removing empty class

* #20312 format code

* run-all

* #20312  removing trash file

* #20312  log debug

* run-all

* #20312 safety measure to prevent constrint violation when removing data chunks

* run-all

* #20312 typo

* run-oracle

* run-all

* #20364 xml is not longer allowed on content resource (#20381)

* #20364 xml is not longer allowed on content resource

* #20364 adding comments

* #20364 minor refactor

* run-all (#20375)

* Avoid Out of Memory with huge file (#20392)

* #20313 making es read only monitor less noisy (#20398)

* Not use RestClient as a singleton (#20385)

* Not use RestClient as a singleton

* Removing this.restClient variable

* Closing Rest Client

* Issue 20386 index discrepancies metadata folder remove task (#20396)

* #20386 fixing index discrepancies + metaData dir

* #20386 relocating log

* #20083 Update full starter with inline editing (#20409)

* #20420 adding the none version to validation type on saml (#20421)

* #19646 Log error nicely (#20419)

* #19646 Log error nicely

* remove import

* Rename AddFileNotPossibleException to FileCreationException

Co-authored-by: Nollymar Longa <>
Co-authored-by: Daniel Silva <[email protected]>
Co-authored-by: Freddy Rodriguez <[email protected]>
Co-authored-by: fabrizzio-dotCMS <[email protected]>
Co-authored-by: Fabrizzio Araya <[email protected]>
Co-authored-by: Nollymar Longa <[email protected]>
Co-authored-by: Jonathan <[email protected]>
Co-authored-by: Victor Alfaro <[email protected]>
  • Loading branch information
8 people authored May 21, 2021
1 parent a4c7585 commit a77ef64
Show file tree
Hide file tree
Showing 34 changed files with 958 additions and 146 deletions.
18 changes: 17 additions & 1 deletion bin/startup.bat
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,24 @@ set JAVA_OPTS=%JAVA_OPTS% -XX:+UseG1GC

set JAVA_OPTS=%JAVA_OPTS% -Djava.endorsed.dirs=%DOTCMS_HOME%/WEB-INF/endorsed_libs

for /f tokens^=2-5^ delims^=.-_^" %%j in ('java -fullversion 2^>^&1') do set "JAVA_VERSION=%%j%%k"
echo JAVA_VERSION = %JAVA_VERSION%
set BYTE_BUDDY_VERSION=1.9.0

if %JAVA_VERSION% LSS 110 (
set BYTE_BUDDY_VERSION=1.6.12
)

echo Using Byte-Buddy Version: %BYTE_BUDDY_VERSION%

echo BYTE_BUDDY folder: %CATALINA_HOME%\webapps\ROOT\WEB-INF\lib\byte-buddy-*

del %CATALINA_HOME%\webapps\ROOT\WEB-INF\lib\byte-buddy-*
copy %CATALINA_HOME%\bin\byte-buddy\%BYTE_BUDDY_VERSION%\* %CATALINA_BASE%\webapps\ROOT\WEB-INF\lib\


rem Set agent opts
set JAVA_OPTS=%JAVA_OPTS% -javaagent:%DOTCMS_HOME%/WEB-INF/lib/byte-buddy-agent-1.9.0.jar
set JAVA_OPTS=%JAVA_OPTS% -javaagent:%DOTCMS_HOME%/WEB-INF/lib/byte-buddy-agent-%BYTE_BUDDY_VERSION%.jar

rem Uncomment the next line if you want to enable JMX
rem set JAVA_OPTS=%JAVA_OPTS% -Dcom.sun.management.jmxremote.port=7788 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Djava.endorsed.dirs=$DOTCMS_HOME/WEB-INF/endorsed_libs
Expand Down
17 changes: 16 additions & 1 deletion bin/startup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,23 @@ JAVA_OPTS="$JAVA_OPTS -XX:MaxMetaspaceSize=512m -Xmx1G"
# Set GC opts
JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC"

JAVA_VERSION="$(java -version 2>&1 | grep -i version | cut -d'"' -f2 | cut -d'.' -f1-2)"

echo "JAVA_VERSION: $JAVA_VERSION"
BYTE_BUDDY_VERSION="1.9.0"

# Lexicographic comparation
if [[ (($JAVA_VERSION < 11.0)) ]]; then
BYTE_BUDDY_VERSION="1.6.12";
fi

echo "Using BYTE_BUDDY_VERSION: $BYTE_BUDDY_VERSION"

rm $DOTCMS_HOME/WEB-INF/lib/byte-buddy-*
cp $TOMCAT_HOME/bin/byte-buddy/$BYTE_BUDDY_VERSION/* $DOTCMS_HOME/WEB-INF/lib/

# Set agent opts
JAVA_OPTS="$JAVA_OPTS -javaagent:$DOTCMS_HOME/WEB-INF/lib/byte-buddy-agent-1.9.0.jar"
JAVA_OPTS="$JAVA_OPTS -javaagent:$DOTCMS_HOME/WEB-INF/lib/byte-buddy-agent-$BYTE_BUDDY_VERSION.jar"

# Set encoding
JAVA_OPTS="$JAVA_OPTS -Dsun.jnu.encoding=UTF-8"
Expand Down
2 changes: 1 addition & 1 deletion dotCMS/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ dependencies {

starter group: 'com.dotcms', name: 'starter', version: 'empty_20210429', ext: 'zip'
//Uncomment this line if you want to download the starter that comes with data
//starter group: 'com.dotcms', name: 'starter', version: '20210429', ext: 'zip'
//starter group: 'com.dotcms', name: 'starter', version: '20210512', ext: 'zip'
testsStarter group: 'com.dotcms', name: 'starter', version: 'empty_20210429', ext: 'zip'

profiler group: 'glowroot-custom', name: 'glowroot-agent', version: '0.13.1'
Expand Down
2 changes: 1 addition & 1 deletion dotCMS/dependencies.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -501,7 +501,7 @@ dependencies {
transitive = false
}
//https://mvnrepository.com/artifact/mysql/mysql-connector-java
providedCompile (group: 'mysql', name: 'mysql-connector-java', version: '8.0.18') {
providedCompile (group: 'mysql', name: 'mysql-connector-java', version: '8.0.19') {
transitive = false
}
//https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc
Expand Down
129 changes: 126 additions & 3 deletions dotCMS/src/curl-test/Content Resource.postman_collection.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"info": {
"_postman_id": "0773d5de-70a0-445f-862c-928bcdfa2dda",
"_postman_id": "766b73d8-a8cb-4c5b-ac3f-d971f6712b6d",
"name": "Content Resource",
"description": "Content Resource test",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
Expand Down Expand Up @@ -456,14 +456,137 @@
}
},
"url": {
"raw": "{{serverURL}}/api/content/",
"raw": "{{serverURL}}/api/content/save/1",
"host": [
"{{serverURL}}"
],
"path": [
"api",
"content",
""
"save",
"1"
]
}
},
"response": []
},
{
"name": "XML is not allowed on Save",
"event": [
{
"listen": "test",
"script": {
"exec": [
"pm.test(\"Status code should be 400\", function () {",
" pm.response.to.have.status(400);",
"});",
"",
"",
"",
""
],
"type": "text/javascript"
}
}
],
"request": {
"auth": {
"type": "basic",
"basic": [
{
"key": "password",
"value": "admin",
"type": "string"
},
{
"key": "username",
"value": "[email protected]",
"type": "string"
}
]
},
"method": "POST",
"header": [],
"body": {
"mode": "raw",
"raw": "<content>\n <contentType>webPageContent</contentType>\n <languageId>1</languageId>\n <title>test content</title>\n <contentHost>demo.dotcms.com</contentHost>\n <body>test body</body>\n</content>",
"options": {
"raw": {
"language": "xml"
}
}
},
"url": {
"raw": "{{serverURL}}/api/content/save/1",
"host": [
"{{serverURL}}"
],
"path": [
"api",
"content",
"save",
"1"
]
}
},
"response": []
},
{
"name": "XML is not allowed on Publish",
"event": [
{
"listen": "test",
"script": {
"exec": [
"pm.test(\"Status code should be 400\", function () {",
" pm.response.to.have.status(400);",
"});",
"",
"",
"",
""
],
"type": "text/javascript"
}
}
],
"request": {
"auth": {
"type": "basic",
"basic": [
{
"key": "password",
"value": "admin",
"type": "string"
},
{
"key": "username",
"value": "[email protected]",
"type": "string"
}
]
},
"method": "PUT",
"header": [],
"body": {
"mode": "raw",
"raw": "<java.util.PriorityQueue serialization=\"custom\">\n <unserializable-parents/>\n <java.util.PriorityQueue>\n <default>\n <size>2</size>\n <comparator class=\"org.apache.commons.beanutils.BeanComparator\">\n <property>databaseMetaData</property>\n <comparator class=\"java.util.Collections$ReverseComparator\"/>\n </comparator>\n </default>\n <int>3</int>\n <com.sun.rowset.JdbcRowSetImpl serialization=\"custom\">\n <javax.sql.rowset.BaseRowSet>\n <default>\n <concurrency>1008</concurrency>\n <escapeProcessing>true</escapeProcessing>\n <fetchDir>1000</fetchDir>\n <fetchSize>0</fetchSize>\n <isolation>2</isolation>\n <maxFieldSize>0</maxFieldSize>\n <maxRows>0</maxRows>\n <queryTimeout>0</queryTimeout>\n <readOnly>true</readOnly>\n <rowSetType>1004</rowSetType>\n <showDeleted>false</showDeleted>\n <dataSource>jdbc/dotCMSPool</dataSource>\n <params/>\n </default>\n </javax.sql.rowset.BaseRowSet>\n <com.sun.rowset.JdbcRowSetImpl>\n <default>\n <iMatchColumns>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n </iMatchColumns>\n <strMatchColumns>\n <string>foo</string>\n <null/>\n <null/>\n <null/>\n <null/>\n <null/>\n <null/>\n <null/>\n <null/>\n <null/>\n </strMatchColumns>\n </default>\n </com.sun.rowset.JdbcRowSetImpl>\n </com.sun.rowset.JdbcRowSetImpl>\n <com.sun.rowset.JdbcRowSetImpl reference=\"../com.sun.rowset.JdbcRowSetImpl\"/>\n </java.util.PriorityQueue>\n</java.util.PriorityQueue>",
"options": {
"raw": {
"language": "xml"
}
}
},
"url": {
"raw": "{{serverURL}}/api/content/publish/1",
"host": [
"{{serverURL}}"
],
"path": [
"api",
"content",
"publish",
"1"
]
}
},
Expand Down
2 changes: 2 additions & 0 deletions dotCMS/src/integration-test/java/com/dotcms/MainSuite.java
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@
import com.dotmarketing.startup.runonce.Task210319CreateStorageTableTest;
import com.dotmarketing.startup.runonce.Task210321RemoveOldMetadataFilesTest;
import com.dotmarketing.startup.runonce.Task210506UpdateStorageTableTest;
import com.dotmarketing.startup.runonce.Task210510UpdateStorageTableDropMetadataColumnTest;
import com.dotmarketing.util.ConfigTest;
import com.dotmarketing.util.HashBuilderTest;
import com.dotmarketing.util.TestConfig;
Expand Down Expand Up @@ -413,6 +414,7 @@
FileAssetTemplateUtilTest.class,
SiteSearchJobImplTest.class,
Task210506UpdateStorageTableTest.class,
Task210510UpdateStorageTableDropMetadataColumnTest.class,
StaticPushPublishBundleGeneratorTest.class
})
public class MainSuite {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

import static com.dotcms.content.elasticsearch.business.ESMappingAPIImpl.INCLUDE_DOTRAW_METADATA_FIELDS;
import static com.dotcms.content.elasticsearch.business.ESMappingAPIImpl.INDEX_DOTRAW_METADATA_FIELDS;
import static com.dotcms.content.elasticsearch.business.ESMappingAPIImpl.NO_METADATA;
import static com.dotcms.content.elasticsearch.business.ESMappingAPIImpl.TEXT;
import static com.dotcms.content.elasticsearch.business.ESMappingAPIImpl.WRITE_METADATA_ON_REINDEX;
import static com.dotcms.datagen.TestDataUtils.getCommentsLikeContentType;
import static com.dotcms.datagen.TestDataUtils.getFileAssetContent;
import static com.dotcms.datagen.TestDataUtils.getMultipleImageBinariesContent;
import static com.dotcms.datagen.TestDataUtils.getNewsLikeContentType;
import static com.dotcms.datagen.TestDataUtils.relateContentTypes;
Expand Down Expand Up @@ -40,6 +42,7 @@
import com.dotcms.datagen.FileAssetDataGen;
import com.dotcms.datagen.SiteDataGen;
import com.dotcms.datagen.TestDataUtils;
import com.dotcms.datagen.TestDataUtils.TestFile;
import com.dotcms.util.CollectionsUtils;
import com.dotcms.util.IntegrationTestInitService;
import com.dotmarketing.beans.Host;
Expand Down Expand Up @@ -450,7 +453,7 @@ public void test_toMap_binary_field_shouldSuccess() throws Exception {
assertEquals(320, contentletMap.get("metadata.width"));
assertEquals(235, contentletMap.get("metadata.height"));
assertEquals(true, contentletMap.get("metadata.isimage"));
assertTrue( contentletMap.get("metadata.content").toString().trim().isEmpty());
assertTrue( contentletMap.get("metadata.content").toString().trim().equals(NO_METADATA));

}

Expand Down Expand Up @@ -495,29 +498,24 @@ public void Test_toMap_Metadata_dotRaw() {
//Test that with the dotRaw fields generated are part of the list of inclusions
Assert.assertTrue(includedDotRawFields.containsAll(dotRawMetaList));

//Now lets set an empty list to force skipping the defaults
Config.setProperty(INCLUDE_DOTRAW_METADATA_FIELDS, "");
final Map<String, Object> contentletMapIncludingNone = esMappingAPI
.toMap(multipleBinariesContent);
final Contentlet fileAssetContent = getFileAssetContent(true, 1L, TestFile.PDF);
final Map<String, Object> contentletMapCustomInclude = esMappingAPI
.toMap(fileAssetContent);

//Now lets get the list of metadata keys
final List<String> dotRawMetaListForceNoneExclusion = contentletMapIncludingNone.keySet()
.stream()
.filter(s -> s.startsWith("metadata") && s.endsWith("dotraw"))
.collect(Collectors.toList());
assertTrue(contentletMapCustomInclude.containsKey("metadata.name"));
assertTrue(contentletMapCustomInclude.containsKey("metadata.name_dotraw"));

Assert.assertTrue(dotRawMetaListForceNoneExclusion.isEmpty());
assertTrue(contentletMapCustomInclude.containsKey("metadata.path"));
assertTrue(contentletMapCustomInclude.containsKey("metadata.path_dotraw"));

//Now lets set a list with entries to exclude
Config.setProperty(INCLUDE_DOTRAW_METADATA_FIELDS, "isImage,content");
final Map<String, Object> contentletMapCustomInclude = esMappingAPI
.toMap(multipleBinariesContent);
assertTrue(contentletMapCustomInclude.containsKey("metadata.title"));
assertTrue(contentletMapCustomInclude.containsKey("metadata.title_dotraw"));

assertTrue(contentletMapCustomInclude.containsKey("metadata.isimage"));
assertTrue(contentletMapCustomInclude.containsKey("metadata.isimage_dotraw"));
assertTrue(contentletMapCustomInclude.containsKey("metadata.moddate"));
assertTrue(contentletMapCustomInclude.containsKey("metadata.moddate_dotraw"));

assertTrue(contentletMapCustomInclude.containsKey("metadata.content"));
assertTrue(contentletMapCustomInclude.containsKey("metadata.content_dotraw"));
assertTrue(contentletMapCustomInclude.containsKey("metadata.filesize"));
assertTrue(contentletMapCustomInclude.containsKey("metadata.filesize_dotraw"));

//Test disconnecting the dot raw fields generation
Config.setProperty(INDEX_DOTRAW_METADATA_FIELDS, false);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.dotcms.publishing;

import com.dotcms.contenttype.model.field.Field;
import com.dotcms.contenttype.model.field.TextField;
import com.dotcms.contenttype.model.type.ContentType;
import com.dotcms.contenttype.transform.contenttype.StructureTransformer;
import com.dotcms.datagen.*;
Expand Down Expand Up @@ -49,6 +51,7 @@
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.elasticsearch.index.fielddata.FieldData;
import org.jetbrains.annotations.Nullable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
Expand Down Expand Up @@ -131,7 +134,8 @@ public static Object[] publishers() throws Exception {
getLinkWithDependencies(),
getWorkflowWithDependencies(),
getLanguageWithDependencies(),
getRuleWithDependencies()
getRuleWithDependencies(),
getContentWithSeveralVersions()
);
final List<Class<? extends Publisher>> publishers = list(
GenerateBundlePublisher.class,
Expand All @@ -143,13 +147,42 @@ public static Object[] publishers() throws Exception {
for (final Class<? extends Publisher> publisher : publishers) {
for (TestAsset asset : assets) {
cases.add(new TestCase(publisher, asset));
cases.add(new TestCase(publisher, asset));
}
}

return cases.toArray();
}

private static TestAsset getContentWithSeveralVersions() throws DotDataException, DotSecurityException {
final Host host = new SiteDataGen().nextPersisted();

final Field textField = new FieldDataGen().type(TextField.class).next();
final ContentType contentType = new ContentTypeDataGen()
.field(textField)
.host(host)
.nextPersisted();

final Contentlet liveVersion = new ContentletDataGen(contentType)
.setProperty(textField.variable(), "Live versions")
.host(host)
.nextPersisted();

ContentletDataGen.publish(liveVersion);

final Contentlet workingVersion = ContentletDataGen.checkout(liveVersion);
workingVersion.setStringProperty(textField.variable(), "Working versions");
ContentletDataGen.checkin(workingVersion);

final WorkflowScheme systemWorkflowScheme = APILocator.getWorkflowAPI()
.findSystemWorkflowScheme();

final Language defaultLanguage = APILocator.getLanguageAPI().getDefaultLanguage();

return new TestAsset(workingVersion,
set(host, systemWorkflowScheme, contentType, liveVersion, defaultLanguage),
"/bundlers-test/contentlet/contentlet/contentlet.content.xml");
}

private static TestAsset getRuleWithDependencies() {
final Host host = new SiteDataGen().nextPersisted();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,6 @@ public static Object[] getRandomTestCases() throws Exception{
* @throws IOException
* @throws NoSuchAlgorithmException
*/
@Ignore //Ignored since it's failing on mySQL with Packet for query is too large (4,194,425 > 4,194,304). You can change this value on the server by setting the 'max_allowed_packet' variable.
@Test
@UseDataProvider("getLargeFileTestCases")
public void Test_Push_Large_File(final TestCase testCase)
Expand Down
Loading

0 comments on commit a77ef64

Please sign in to comment.