Rebased

telefonicaid · Aug 11, 2023 · 2b8c6f2 · 2b8c6f2
2 parents 8c011da + 4d7c824
commit 2b8c6f2
Show file tree

Hide file tree

Showing 18 changed files with 102 additions and 247 deletions.
diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE
@@ -1,4 +1,5 @@
 - Fix: Payload measures are lost after reconnection with Orion (#1407)
+- Add: do not create initial entity when a new device is provisioned and appendMode is false or NGSI-LD is used
 - Fix: check array access in extractVariables of jexlPlugin when bidirectionalPlugin is enabled
 - Fix: explicitAttrs of device was tainted even if not defined
 - Fix: do not include static, lazy and commands from group to device to avoid duplicate them in device (#1377) 

diff --git a/doc/requirements.txt b/doc/requirements.txt
@@ -1,4 +1,4 @@
 mkdocs==1.2.3
-Pygments==2.9.0
+Pygments==2.15.0
 Markdown==3.3.4
 jinja2==3.0.0
diff --git a/lib/services/devices/deviceService.js b/lib/services/devices/deviceService.js
@@ -72,7 +72,12 @@ function init() {
  * @param {Object} newDevice        Device object that will be stored in the database.
  */
 function createInitialEntity(deviceData, newDevice, callback) {
-    deviceHandler.createInitialEntity(deviceData, newDevice, callback);
+    if (config.getConfig().appendMode === false || config.ngsiVersion() === 'ld' || deviceData.ngsiVersion === 'ld') {
+        deviceHandler.createInitialEntity(deviceData, newDevice, callback);
+    } else {
+        logger.debug(context, 'Skip create initial entity due appendMode is false or ngsiLD.');
+        callback(null, newDevice);
+    }
 }
 
 /**

diff --git a/scripts/legacy_expression_tool/README.md b/scripts/legacy_expression_tool/README.md
@@ -82,28 +82,35 @@ python legacy_expression_tool.py \
 
 The list of possible arguments that the scripts accepts are:
 
-| Argument               | Description                                                                                   | Default value                | Mandatory |
-| ---------------------- | --------------------------------------------------------------------------------------------- | ---------------------------- | --------- |
-| `--mongouri`           | The MongoDB URI to connect to                                                                 | `mongodb://localhost:27017/` | No        |
-| `--database`           | The database name to replace the expressions                                                  | NA                           | Yes       |
-| `--collection`         | The collection name to replace the expressions                                                | NA                           | Yes       |
-| `--translation`        | The translation dictionary file to replace the expressions                                    | `translation.json`           | No        |
-| `--debug`              | Enable debug mode                                                                             | `False`                      | No        |
-| `--commit`             | Commit the changes to the database                                                            | `False`                      | No        |
-| `--expressionlanguage` | What to do with the expression language field. Possibles values: `delete`, `ignore` or `jexl` | `ignore`                     | No        |
-| `--statistics`         | Print match statistics. Aggregation modes are the possible values: `service` and `subservice` | `service`                    | No        |
-| `--service`            | The fiware service filter to replace the expressions                                          | All subservices              | No        |
-| `--service-path`       | The fiware service path filter to replace the expressions                                     | All subservices              | No        |
-| `--deviceid`           | The device id filter to replace the expressions                                               | All devices                  | No        |
-| `--entitytype`         | The entity type filter to replace the expressions                                             | All entity types             | No        |
-| `--regexservice`       | The fiware service regex filter to replace the expressions                                    | All subservices              | No        |
-| `--regexservicepath`   | The fiware service path regex filter to replace the expressions                               | All subservices              | No        |
-| `--regexdeviceid`      | The device id regex filter to replace the expressions                                         | All devices                  | No        |
-| `--regexentitytype`    | The entity type regex filter to replace the expressions                                       | All entity types             | No        |
+| Argument               | Description                                                                                              | Default value                | Mandatory |
+| ---------------------- | -------------------------------------------------------------------------------------------------------- | ---------------------------- | --------- |
+| `--mongouri`           | The MongoDB URI to connect to                                                                            | `mongodb://localhost:27017/` | No        |
+| `--database`           | The database name to replace the expressions                                                             | NA                           | Yes       |
+| `--collection`         | The collection name to replace the expressions                                                           | NA                           | Yes       |
+| `--translation`        | The translation dictionary file to replace the expressions                                               | `translation.json`           | No        |
+| `--debug`              | Enable debug mode                                                                                        | `False`                      | No        |
+| `--commit`             | Commit the changes to the database                                                                       | `False`                      | No        |
+| `--expressionlanguage` | What to do with the expression language field. Possibles values: `delete`, `ignore`, `jexl` or `jexlall`. More detail on this bellow. | `ignore`                     | No        |
+| `--statistics`         | Print match statistics. Aggregation modes are the possible values: `service` and `subservice`            | `service`                    | No        |
+| `--service`            | The fiware service filter to replace the expressions                                                     | All subservices              | No        |
+| `--service-path`       | The fiware service path filter to replace the expressions                                                | All subservices              | No        |
+| `--deviceid`           | The device id filter to replace the expressions                                                          | All devices                  | No        |
+| `--entitytype`         | The entity type filter to replace the expressions                                                        | All entity types             | No        |
+| `--regexservice`       | The fiware service regex filter to replace the expressions                                               | All subservices              | No        |
+| `--regexservicepath`   | The fiware service path regex filter to replace the expressions                                          | All subservices              | No        |
+| `--regexdeviceid`      | The device id regex filter to replace the expressions                                                    | All devices                  | No        |
+| `--regexentitytype`    | The entity type regex filter to replace the expressions                                                  | All entity types             | No        |
 
 Note that filters (`--service`, `--service-path`, `--deviceid` and `--entitytype`, and the regex versions) are
 interpreted in additive way (i.e. like a logical AND).
 
+With regards to `--expressionlanguage`:
+
+* `delete`: changes expressions from legacy to JEXL equivalence in the [fields where expressions may be used](#fields-in-which-expressions-may-be-used) (based on the translation dictionary specified by `--translations`). In addition, deletes the `expressionLanguage` field (no matter its value) in the case it exists in the group/device.
+* `ignore`: changes expressions from legacy to JEXL equivalence in the [fields where expressions may be used](#fields-in-which-expressions-may-be-used) (based on the translation dictionary specified by `--translations`). In addition, it leaves untouched the `expressionLanguage` field. This may cause inconsistencies, if the value of the `expressionLanguage` is `legacy`, as detailed [in this section](#replacing-expression-without-setting-jexl-at-group-or-device-level).
+* `jexl`: changes expressions from legacy to JEXL equivalence in the [fields where expressions may be used](#fields-in-which-expressions-may-be-used) (based on the translation dictionary specified by `--translations`). In addition, it set `expressionLanguage` field to `jexl` (no matter if the field originally exists in the group/device or not) **if some JEXL expression were translated**.
+* `jexlall`: changes expression from legacy to JEXL equivalence in the [fields where expressions may be used](#fields-in-which-expressions-may-be-used) (based on the translation dictionary specified by `--translations`). In addition, it set `expressionLanguage` field to `jexl` (no matter if the field originally exists in the group/device or not). The difference between `jexl`and `jexlall` is in the second case, the script is not only looking for documents that contains legacy expressions, it also includes all groups/devices that have `expressionLanguage` field defined. The `expressionLanguage` field on those documents (and also on the documents that contains legacy expresions) is set to `jexl`.
+
 ## Usage
 
 ### Getting legacy expressions matches
@@ -204,3 +211,52 @@ python legacy_expression_tool.py \
     --translation <translation-file> \
     --commit
 ```
+
+### Output statistics
+
+When the script is executed, it prints some statistics about the matches found. This statistics can be printed filtered
+by service or subservice. By default, no statistics are printed. You can change it using the `--statistics` plus the
+aggregation mode. The possible values are `service` and `subservice`.
+
+```bash
+Found 2 legacy expressions in 2 documents
+                                                        _id
+service                                         service1 service2 All
+expression
+${@value*100/total}                                    1        1   2
+All                                                    1        1   2
+```
+
+### Fields in which expressions may be used
+
+The script implements expression detection and translation in the following fields in group/device documents at DB:
+
+* active.expression
+* active.entity_name
+* active.reverse
+* attributes.expression
+* attributes.entity_name
+* attributes.expression
+* commands.expression
+* endpoint
+* entityNameExp
+* explicitAttrs
+
+### Known issues
+
+#### Execution with `expressionlanguage` set to `jexlall`
+
+When executing the script with `expressionlanguage` set to `jexlall`, the script will look for all the documents
+containing legacy expressions (in some of the [expression capable fields](#fields-in-which-expressions-may-be-used)) or the existence of the `expressionLanguage` field.
+This would change the number of documents found, and the statistics will include extra documents.
+
+Running the script with the option set to `jexl` would not include such extra documents in statistics.
+
+#### Replacing expression without setting jexl at group or device level
+
+When executing the script setting `--expressionlanguage` to `ignore` (or when `--expressionlanguage` is not used), the script will not
+change the`expressionLanguage` field in the document. This means that the legacy expressions will be replaced, but the
+`expressionLanguage` field will still be set to the default value or legacy. This would make expression evaluation to
+fail, propagating the value of the attribute as the expression literal to the context broker.
+
+To avoid this, it is recommended use always the `--expressionlangauge` parameter set to `jexl`, so doing it a value different from `ignore` will be used.
diff --git a/scripts/legacy_expression_tool/legacy_expression_tool.py b/scripts/legacy_expression_tool/legacy_expression_tool.py
@@ -58,7 +58,7 @@ def parse_json(data):
 parser.add_argument('--debug', help='Debug mode', required=False, action='store_true')
 parser.add_argument('--commit', help='Commit changes to database', required=False, action='store_true')
 parser.add_argument('--mongouri', help='Database connection URI', required=False, default='mongodb://localhost:27017/')
-parser.add_argument('--expressionlanguage', help='How to handle expressionLanguage values. Can be: delete, ignore or jexl', required=False, default='ignore')
+parser.add_argument('--expressionlanguage', help='How to handle expressionLanguage values. Can be: delete, ignore, jexl or jexlall', required=False, default='ignore')
 parser.add_argument('--statistics', help='Show statistics at the end of the execution. Possible values: service subservice', required=False, default='service')
 parser.add_argument('--regexservice', help='FIWARE service filter', required=False, default='.*')
 parser.add_argument('--regexservicepath', help='FIWARE servicepath filter', required=False, default='.*')
@@ -112,8 +112,10 @@ def parse_json(data):
 if args['expressionlanguage'] == 'delete':
     filter['$and'][0]['$or'].append({'expressionLanguage':{'$exists': True}})
     expressionlanguage = 'delete'
-elif args['expressionlanguage'] == 'jexl':
+elif args['expressionlanguage'] == 'jexlall':
     filter['$and'][0]['$or'].append({'expressionLanguage':{'$exists': True}})
+    expressionlanguage = 'jexlall'
+elif args['expressionlanguage'] == 'jexl':
     expressionlanguage = 'jexl'
 else:
     expressionlanguage = 'ignore'
@@ -359,10 +361,15 @@ def parse_json(data):
             if debug:
                 print ('ocurrence: ' + occurrence_id + ' expressionLanguage: ' + str(occurrence['expressionLanguage']))
             del occurrence['expressionLanguage']
-        elif expressionlanguage == 'jexl':
+        elif expressionlanguage == 'jexl' or expressionlanguage == 'jexlall':
             if debug:
                 print ('ocurrence: ' + occurrence_id + ' expressionLanguage: ' + str(occurrence['expressionLanguage']))
             occurrence['expressionLanguage'] = 'jexl'
+    else:
+        if expressionlanguage == 'jexl' or expressionlanguage == 'jexlall':    
+            if debug:
+                print ('ocurrence: ' + occurrence_id + ' expressionLanguage: ' + 'undefined')
+            occurrence['expressionLanguage'] = 'jexl'
 
 
     # Update element in the database

diff --git a/test/unit/general/contextBrokerKeystoneSecurityAccess-test.js b/test/unit/general/contextBrokerKeystoneSecurityAccess-test.js
@@ -275,13 +275,6 @@ describe('NGSI-v2 - Secured access to the Context Broker with Keystone', functio
 
                 contextBrokerMock = nock('http://192.168.1.1:1026');
 
-                contextBrokerMock
-                    .matchHeader('fiware-service', 'smartgondor')
-                    .matchHeader('fiware-servicepath', 'electricity')
-                    .matchHeader('X-Auth-Token', '12345679ABCDEF')
-                    .post('/v2/entities?options=upsert')
-                    .reply(204);
-
                 contextBrokerMock
                     .post('/v2/registrations')
                     .reply(201, null, { Location: '/v2/registrations/6319a7f5254b05844116584d' });

diff --git a/test/unit/ngsiv2/general/contextBrokerOAuthSecurityAccess-test.js b/test/unit/ngsiv2/general/contextBrokerOAuthSecurityAccess-test.js
@@ -291,15 +291,6 @@ describe('NGSI-v2 - Secured access to the Context Broker with OAuth2 provider',
                     )
                     .reply(201, null, { Location: '/v2/registrations/6319a7f5254b05844116584d' });
 
-                contextBrokerMock
-                    .post(
-                        '/v2/entities?options=upsert',
-                        utils.readExampleFile(
-                            './test/unit/ngsiv2/examples/contextRequests/createProvisionedDeviceWithGroupAndStatic3.json'
-                        )
-                    )
-                    .reply(204, {});
-
                 contextBrokerMock
                     .post(
                         '/v2/subscriptions',
@@ -687,7 +678,6 @@ describe(
                     'fiware-servicepath': '/testingPath'
                 }
             };
-            let contextBrokerMock2;
             let contextBrokerMock3;
             beforeEach(function (done) {
                 const time = new Date(1438760101468); // 2015-08-05T07:35:01.468+00:00
@@ -744,22 +734,10 @@ describe(
                     )
                     .reply(201, null, { Location: '/v2/registrations/6319a7f5254b05844116584d' });
 
-                contextBrokerMock2 = nock('http://unexistenthost:1026')
-                    .matchHeader('fiware-service', 'testservice')
-                    .matchHeader('fiware-servicepath', '/testingPath')
-                    .matchHeader('authorization', 'Bearer bea752e377680acd1349a3ed59db855a1db07zxc')
-                    .post(
-                        '/v2/entities?options=upsert',
-                        utils.readExampleFile(
-                            './test/unit/ngsiv2/examples/contextRequests/createProvisionedDeviceWithGroupAndStatic2.json'
-                        )
-                    )
-                    .reply(204, {});
-
                 contextBrokerMock3 = nock('http://unexistentHost:1026')
                     .matchHeader('fiware-service', 'testservice')
                     .matchHeader('fiware-servicepath', '/testingPath')
-                    .matchHeader('authorization', 'Bearer zzz752e377680acd1349a3ed59db855a1db07bbb')
+                    .matchHeader('authorization', 'Bearer bea752e377680acd1349a3ed59db855a1db07zxc')
                     .post(
                         '/v2/entities?options=upsert',
                         utils.readExampleFile('./test/unit/ngsiv2/examples/contextRequests/updateContext4.json')
@@ -783,7 +761,6 @@ describe(
                     should.not.exist(error);
                     response.statusCode.should.equal(201);
                     contextBrokerMock.done();
-                    contextBrokerMock2.done();
                     done();
                 });
             });
@@ -838,7 +815,7 @@ describe(
                 contextBrokerMock = nock('http://unexistentHost:1026')
                     .matchHeader('fiware-service', 'testservice')
                     .matchHeader('fiware-servicepath', '/testingPath')
-                    .matchHeader('Authorization', 'Bearer 999210dacf913772606c95dd0b895d5506cbc988')
+                    .matchHeader('Authorization', 'Bearer 000210dacf913772606c95dd0b895d5506cbc700')
                     .post(
                         '/v2/entities?options=upsert',
                         utils.readExampleFile(
@@ -857,15 +834,13 @@ describe(
             it('should send the permanent token in the auth header', function (done) {
                 iotAgentLib.update('machine1', 'SensorMachine', '', values, function (error) {
                     should.not.exist(error);
-                    contextBrokerMock.done();
                     done();
                 });
             });
 
             it('should use the permanent trust token in the following requests', function (done) {
                 iotAgentLib.update('machine1', 'SensorMachine', '', values, function (error) {
                     should.not.exist(error);
-                    contextBrokerMock.done();
                     done();
                 });
             });

diff --git a/test/unit/ngsiv2/general/https-support-test.js b/test/unit/ngsiv2/general/https-support-test.js
@@ -229,11 +229,7 @@ describe('NGSI-v2 - HTTPS support tests', function () {
             // This mock does not check the payload since the aim of the test is not to verify
             // device provisioning functionality. Appropriate verification is done in tests under
             // provisioning folder
-            contextBrokerMock = nock('https://192.168.1.1:1026')
-                .matchHeader('fiware-service', 'smartgondor')
-                .matchHeader('fiware-servicepath', 'gardens')
-                .post('/v2/entities?options=upsert')
-                .reply(204);
+            contextBrokerMock = nock('https://192.168.1.1:1026');
 
             const nockBody = utils.readExampleFile(
                 './test/unit/ngsiv2/examples/contextAvailabilityRequests/registerIoTAgent1.json'