From 697ca68083cc8132ac08925f1ce63b2c7ecc3ea8 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 3 Apr 2024 15:55:51 +0200 Subject: [PATCH 01/83] add test case --- test/functional/testCases.js | 94 +++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 5d558c7c3..d23ee28ea 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -1036,7 +1036,8 @@ const testCases = [ ] }, { - describeName: '0140 - Simple group with active attribute + chained JEXL expression text (a | trim | replacestr("hello","hi"))', + describeName: + '0140 - Simple group with active attribute + chained JEXL expression text (a | trim | replacestr("hello","hi"))', provision: { url: 'http://localhost:' + config.iota.server.port + '/iot/services', method: 'POST', @@ -1976,6 +1977,97 @@ const testCases = [ } ] }, + { + describeName: '0430 - Simple group with active attribute + timestamp mapping defined', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + timestamp: true, + entity_type: globalEnv.entity_type, + commands: [], + lazy: [], + attributes: [ + { + object_id: 'mydatetime', + name: 'TimeInstant', + type: 'DateTime' + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending a measure through http IT should map the measure to timestamp attribute and use it for timestmap sent to Context Broker', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + mydatetime: '2022-02-02T02:22:22.222Z' + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + }, + { + shouldName: + 'A - WHEN sending a measure through http IT should map the measure to timestamp attribute and use it for timestmap and other metadata attributes sent to Context Broker', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + mydatetime: '2022-02-02T02:22:22.222Z', + a: 23 + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + a: { + value: 23, + type: 'Text', + metadata: { + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + }, + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + } + ] + }, // 0500 - EXPLICIT ATTRIBUTES TESTS { describeName: '0500 - Group with explicit attrs:false (boolean) + active atributes', From 61b3c236b866c55ad1437e363d5989f242aeca25 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 3 Apr 2024 16:37:56 +0200 Subject: [PATCH 02/83] check if timestamp is a mapped attribute --- lib/services/ngsi/entities-NGSI-v2.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 38d0a1471..42dfb87a8 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -278,6 +278,7 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call let jexlctxt = {}; //will store the whole context (not just for JEXL) let payload = {}; //will store the final payload let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. + let timestampAttr = null; let plainMeasures = null; //will contain measures POJO let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); @@ -308,15 +309,14 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call jexlctxt = reduceAttrToPlainObject(idTypeSSSList, jexlctxt); //Managing timestamp (mustInsertTimeInstant flag to decide if we should insert Timestamp later on) - const mustInsertTimeInstant = - typeInformation.timestamp !== undefined - ? typeInformation.timestamp - : false; + const mustInsertTimeInstant = typeInformation.timestamp !== undefined ? typeInformation.timestamp : false; if (mustInsertTimeInstant) { //remove TimeInstant from measures measures = measures.filter((item) => item.name !== constants.TIMESTAMP_ATTRIBUTE); - + if (typeInformation && typeInformation.active) { + timestampAttr = typeInformation.active.filter((item) => item.name === constants.TIMESTAMP_ATTRIBUTE); + } if (plainMeasures[constants.TIMESTAMP_ATTRIBUTE]) { //if it comes from a measure if (moment(plainMeasures[constants.TIMESTAMP_ATTRIBUTE], moment.ISO_8601, true).isValid()) { @@ -327,6 +327,16 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call ); return; } + } else if (timestampAttr && timestampAttr.length > 0 && plainMeasures[timestampAttr[0]['object_id']]) { + // Maybe TimeInstant is a mapped attribute + if (moment(plainMeasures[timestampAttr[0]['object_id']], moment.ISO_8601, true).isValid()) { + timestamp.value = plainMeasures[timestampAttr[0]['object_id']]; + } else { + callback( + new errors.BadTimestamp(plainMeasures[timestampAttr[0]['object_id']], entityName, typeInformation) + ); + return; + } } else if (!typeInformation.timezone) { timestamp.value = new Date().toISOString(); jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; From a1014271f2dc604769aef9a322ab45a13c93c5d2 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 4 Apr 2024 09:47:57 +0200 Subject: [PATCH 03/83] update doc --- doc/api.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/api.md b/doc/api.md index 8849cb276..6b5410d43 100644 --- a/doc/api.md +++ b/doc/api.md @@ -133,9 +133,9 @@ parameters. The specific parameters that can be configured for a given config gr ### Devices A device contains the information that connects a physical device to a particular entity in the Context Broker. Devices -are identified by a `device_id`, and they are associated to an existing config group based in `apiKey` matching or -`type` matching (in the case `apiKey` matching fails). For instance, let's consider a situation in which a config group -has been provisioned with `type=X`/`apiKey=111` and no other config group has been provisioned. +are identified by a `device_id`, and they are associated to an existing config group based in `apiKey` matching. For +instance, let's consider a situation in which a config group has been provisioned with `type=X`/`apiKey=111` and no +other config group has been provisioned. The IoT Agents offer a provisioning API where devices can be preregistered, so all the information about service and subservice mapping, security information and attribute configuration can be specified in a per device way instead of From bfe95a3dfa3cc5352546dd32eae16dc01a358cfd Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 4 Apr 2024 10:56:17 +0200 Subject: [PATCH 04/83] add new tests --- test/functional/testCases.js | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index d23ee28ea..f927e7d60 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2065,6 +2065,40 @@ const testCases = [ type: 'DateTime' } } + }, + { + shouldName: + 'A - WHEN sending a measure without timestamp through http IT should use system timestamp for mapped attribute and use it for timestmap and other metadata attributes sent to Context Broker', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 23 + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + a: { + value: 23, + type: 'Text', + metadata: { + TimeInstant: { + value: _.isDateString, + type: 'DateTime' + } + } + }, + TimeInstant: { + value: _.isDateString, + type: 'DateTime' + } + } } ] }, From 1d3092af76e8e32b36df964c39c448feb5bee2f6 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 4 Apr 2024 11:15:06 +0200 Subject: [PATCH 05/83] update test --- lib/services/ngsi/entities-NGSI-v2.js | 17 ++++++++++------- test/functional/testCases.js | 1 + 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 42dfb87a8..32b0c3c86 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -278,7 +278,7 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call let jexlctxt = {}; //will store the whole context (not just for JEXL) let payload = {}; //will store the final payload let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. - let timestampAttr = null; + let timestampAttrs = null; //list of mapped TimeInstant attributes let plainMeasures = null; //will contain measures POJO let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); @@ -314,8 +314,11 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call if (mustInsertTimeInstant) { //remove TimeInstant from measures measures = measures.filter((item) => item.name !== constants.TIMESTAMP_ATTRIBUTE); + //search for a Timestamp mapped in an attribute if (typeInformation && typeInformation.active) { - timestampAttr = typeInformation.active.filter((item) => item.name === constants.TIMESTAMP_ATTRIBUTE); + timestampAttrs = typeInformation.active.filter( + (item) => item.name === constants.TIMESTAMP_ATTRIBUTE && item.object_id !== null + ); } if (plainMeasures[constants.TIMESTAMP_ATTRIBUTE]) { //if it comes from a measure @@ -327,13 +330,13 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call ); return; } - } else if (timestampAttr && timestampAttr.length > 0 && plainMeasures[timestampAttr[0]['object_id']]) { - // Maybe TimeInstant is a mapped attribute - if (moment(plainMeasures[timestampAttr[0]['object_id']], moment.ISO_8601, true).isValid()) { - timestamp.value = plainMeasures[timestampAttr[0]['object_id']]; + } else if (timestampAttrs && timestampAttrs.length > 0 && plainMeasures[timestampAttrs[0]['object_id']]) { + // Maybe TimeInstant is a mapped attribute, but just the first one + if (moment(plainMeasures[timestampAttrs[0]['object_id']], moment.ISO_8601, true).isValid()) { + timestamp.value = plainMeasures[timestampAttrs[0]['object_id']]; } else { callback( - new errors.BadTimestamp(plainMeasures[timestampAttr[0]['object_id']], entityName, typeInformation) + new errors.BadTimestamp(plainMeasures[timestampAttrs[0]['object_id']], entityName, typeInformation) ); return; } diff --git a/test/functional/testCases.js b/test/functional/testCases.js index f927e7d60..28f0181fb 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2070,6 +2070,7 @@ const testCases = [ shouldName: 'A - WHEN sending a measure without timestamp through http IT should use system timestamp for mapped attribute and use it for timestmap and other metadata attributes sent to Context Broker', type: 'single', + isRegex: true, measure: { url: 'http://localhost:' + config.http.port + '/iot/json', method: 'POST', From c36ef984368bc886298866c69a0073d726409450 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 4 Apr 2024 13:26:32 +0200 Subject: [PATCH 06/83] add case for a bad timestamp when timestamp is an attribute mapped --- lib/services/ngsi/entities-NGSI-v2.js | 8 +++- test/functional/testCases.js | 57 +++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 32b0c3c86..d9a01629d 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -332,7 +332,13 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call } } else if (timestampAttrs && timestampAttrs.length > 0 && plainMeasures[timestampAttrs[0]['object_id']]) { // Maybe TimeInstant is a mapped attribute, but just the first one - if (moment(plainMeasures[timestampAttrs[0]['object_id']], moment.ISO_8601, true).isValid()) { + if (timestampAttrs[0]['expression']) { + timestamp.value = expressionPlugin.applyExpression( + timestampAttrs[0]['expression'], + jexlctxt, + typeInformation + ); + } else if (moment(plainMeasures[timestampAttrs[0]['object_id']], moment.ISO_8601, true).isValid()) { timestamp.value = plainMeasures[timestampAttrs[0]['object_id']]; } else { callback( diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 28f0181fb..5153eec7b 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2103,6 +2103,63 @@ const testCases = [ } ] }, + { + describeName: '0431 - Simple group with active attribute + bad timestamp mapping defined', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + timestamp: true, + entity_type: globalEnv.entity_type, + commands: [], + lazy: [], + attributes: [ + { + object_id: 'mydatetime', + name: 'TimeInstant', + type: 'DateTime', + expression: '"2033-03-03T" + "03:33:33.333Z"' + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending a measure through http IT should map the measure to fixed timestamp attribute and use it for timestmap sent to Context Broker', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + mydatetime: '2022-02-02T02:22:22.222Z' + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + TimeInstant: { + value: '2033-03-03T03:33:33.333Z', + type: 'DateTime' + } + } + } + ] + }, // 0500 - EXPLICIT ATTRIBUTES TESTS { describeName: '0500 - Group with explicit attrs:false (boolean) + active atributes', From 7c1fb9eb0246a4fc51e308462c94f1ef04b13b99 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 4 Apr 2024 13:29:12 +0200 Subject: [PATCH 07/83] Update CNR --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e69de29bb..7490ae7a5 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -0,0 +1 @@ +- Fix timestamp attribute mapped cases (#1557) From 2514fbba7d8c9b298e96b60e7facf6563a1d17ca Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 5 Apr 2024 13:10:59 +0200 Subject: [PATCH 08/83] add new case when measure timestamp is send with a attributed mapped to timestamp --- lib/services/ngsi/entities-NGSI-v2.js | 23 +++++++++-------- test/functional/testCases.js | 36 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index d9a01629d..e15ada2d5 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -320,17 +320,8 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call (item) => item.name === constants.TIMESTAMP_ATTRIBUTE && item.object_id !== null ); } - if (plainMeasures[constants.TIMESTAMP_ATTRIBUTE]) { - //if it comes from a measure - if (moment(plainMeasures[constants.TIMESTAMP_ATTRIBUTE], moment.ISO_8601, true).isValid()) { - timestamp.value = plainMeasures[constants.TIMESTAMP_ATTRIBUTE]; - } else { - callback( - new errors.BadTimestamp(plainMeasures[constants.TIMESTAMP_ATTRIBUTE], entityName, typeInformation) - ); - return; - } - } else if (timestampAttrs && timestampAttrs.length > 0 && plainMeasures[timestampAttrs[0]['object_id']]) { + + if (timestampAttrs && timestampAttrs.length > 0 && plainMeasures[timestampAttrs[0]['object_id']]) { // Maybe TimeInstant is a mapped attribute, but just the first one if (timestampAttrs[0]['expression']) { timestamp.value = expressionPlugin.applyExpression( @@ -346,6 +337,16 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call ); return; } + } else if (plainMeasures[constants.TIMESTAMP_ATTRIBUTE]) { + //if it comes from a measure + if (moment(plainMeasures[constants.TIMESTAMP_ATTRIBUTE], moment.ISO_8601, true).isValid()) { + timestamp.value = plainMeasures[constants.TIMESTAMP_ATTRIBUTE]; + } else { + callback( + new errors.BadTimestamp(plainMeasures[constants.TIMESTAMP_ATTRIBUTE], entityName, typeInformation) + ); + return; + } } else if (!typeInformation.timezone) { timestamp.value = new Date().toISOString(); jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 5153eec7b..25cbe2ae0 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2100,6 +2100,42 @@ const testCases = [ type: 'DateTime' } } + }, + { + shouldName: + 'A - WHEN sending a measure with timestamp through http IT should map the measure to timestamp attribute and use it for timestmap and other metadata attributes sent to Context Broker', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + mydatetime: '2022-02-02T02:22:22.222Z', + TimeInstant: '2033-03-03T03:33:33.333Z', + a: 23 + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + a: { + value: 23, + type: 'Text', + metadata: { + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + }, + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } } ] }, From 79a406a64d331a7d5ec48e8a7964ac1268d2f6b5 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 8 Apr 2024 10:36:45 +0200 Subject: [PATCH 09/83] update tests with explicitAttrs case --- test/functional/testCases.js | 170 ++++++++++++++++++++++++++++++++++- 1 file changed, 169 insertions(+), 1 deletion(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 25cbe2ae0..d87005902 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2140,7 +2140,175 @@ const testCases = [ ] }, { - describeName: '0431 - Simple group with active attribute + bad timestamp mapping defined', + describeName: '0431 - Simple group with active attribute + timestamp mapping defined + explicitAttrs', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + timestamp: true, + explicitAttrs: true, + entity_type: globalEnv.entity_type, + commands: [], + lazy: [], + attributes: [ + { + object_id: 'mydatetime', + name: 'TimeInstant', + type: 'DateTime' + }, + { + object_id: 'a', + name: 'a', + type: 'Text' + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending a measure through http IT should map the measure to timestamp attribute and use it for timestmap sent to Context Broker', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + mydatetime: '2022-02-02T02:22:22.222Z' + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + }, + { + shouldName: + 'A - WHEN sending a measure through http IT should map the measure to timestamp attribute and use it for timestmap and other metadata attributes sent to Context Broker', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + mydatetime: '2022-02-02T02:22:22.222Z', + a: 23 + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + a: { + value: 23, + type: 'Text', + metadata: { + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + }, + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + }, + { + shouldName: + 'A - WHEN sending a measure without timestamp through http IT should use system timestamp for mapped attribute and use it for timestmap and other metadata attributes sent to Context Broker', + type: 'single', + isRegex: true, + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 23 + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + a: { + value: 23, + type: 'Text', + metadata: { + TimeInstant: { + value: _.isDateString, + type: 'DateTime' + } + } + }, + TimeInstant: { + value: _.isDateString, + type: 'DateTime' + } + } + }, + { + shouldName: + 'A - WHEN sending a measure with timestamp through http IT should map the measure to timestamp attribute and use it for timestmap and other metadata attributes sent to Context Broker', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + mydatetime: '2022-02-02T02:22:22.222Z', + TimeInstant: '2033-03-03T03:33:33.333Z', + a: 23 + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + a: { + value: 23, + type: 'Text', + metadata: { + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + }, + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + } + ] + }, + { + describeName: '0432 - Simple group with active attribute + bad timestamp mapping defined', provision: { url: 'http://localhost:' + config.iota.server.port + '/iot/services', method: 'POST', From 9e3e7bba9e30789e0ecbeff50f32394a6598b230 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 8 Apr 2024 13:25:32 +0200 Subject: [PATCH 10/83] refactor timestamp procesing --- lib/services/ngsi/entities-NGSI-v2.js | 104 ++++++++++---------------- test/functional/testCases.js | 61 +++------------ 2 files changed, 53 insertions(+), 112 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index e15ada2d5..bc4d5710d 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -311,51 +311,6 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call //Managing timestamp (mustInsertTimeInstant flag to decide if we should insert Timestamp later on) const mustInsertTimeInstant = typeInformation.timestamp !== undefined ? typeInformation.timestamp : false; - if (mustInsertTimeInstant) { - //remove TimeInstant from measures - measures = measures.filter((item) => item.name !== constants.TIMESTAMP_ATTRIBUTE); - //search for a Timestamp mapped in an attribute - if (typeInformation && typeInformation.active) { - timestampAttrs = typeInformation.active.filter( - (item) => item.name === constants.TIMESTAMP_ATTRIBUTE && item.object_id !== null - ); - } - - if (timestampAttrs && timestampAttrs.length > 0 && plainMeasures[timestampAttrs[0]['object_id']]) { - // Maybe TimeInstant is a mapped attribute, but just the first one - if (timestampAttrs[0]['expression']) { - timestamp.value = expressionPlugin.applyExpression( - timestampAttrs[0]['expression'], - jexlctxt, - typeInformation - ); - } else if (moment(plainMeasures[timestampAttrs[0]['object_id']], moment.ISO_8601, true).isValid()) { - timestamp.value = plainMeasures[timestampAttrs[0]['object_id']]; - } else { - callback( - new errors.BadTimestamp(plainMeasures[timestampAttrs[0]['object_id']], entityName, typeInformation) - ); - return; - } - } else if (plainMeasures[constants.TIMESTAMP_ATTRIBUTE]) { - //if it comes from a measure - if (moment(plainMeasures[constants.TIMESTAMP_ATTRIBUTE], moment.ISO_8601, true).isValid()) { - timestamp.value = plainMeasures[constants.TIMESTAMP_ATTRIBUTE]; - } else { - callback( - new errors.BadTimestamp(plainMeasures[constants.TIMESTAMP_ATTRIBUTE], entityName, typeInformation) - ); - return; - } - } else if (!typeInformation.timezone) { - timestamp.value = new Date().toISOString(); - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; - } else { - timestamp.value = moment().tz(typeInformation.timezone).format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; - } - } - logger.debug( context, 'sendUpdateValueNgsi2 called with: entityName=%s, measures=%j, typeInformation=%j, initial jexlContext=%j, timestamp=%j with value=%j', @@ -498,15 +453,6 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call currentAttr.hitted = hitted; currentAttr.value = valueExpression; - - //add TimeInstant to attr metadata - if (mustInsertTimeInstant) { - if (!currentAttr.metadata) { - currentAttr.metadata = {}; - } - currentAttr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; - } - //store de New Attributte in entity data structure if (hitted === true) { if (entities[attrEntityName] === undefined) { @@ -547,17 +493,47 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call //more mesures may be added to the attribute list (unnhandled/left mesaures) l if (explicit === false && Object.keys(measures).length > 0) { - //add Timestamp to measures if needed - if (mustInsertTimeInstant) { - for (let currentMeasure of measures) { - if (!currentMeasure.metadata) { - currentMeasure.metadata = {}; + entities[entityName][typeInformation.type] = entities[entityName][typeInformation.type].concat(measures); + } + + if (mustInsertTimeInstant) { + // search timestamp just in entity attrs + for (let ename in entities) { + for (let etype in entities[ename]) { + timestampAttrs = entities[ename][etype].filter((item) => item.name === constants.TIMESTAMP_ATTRIBUTE); + if (timestampAttrs && timestampAttrs.length > 0) { + timestamp.value = timestampAttrs[0]['value']; + } + } + } + if (timestamp.value) { + if (!moment(timestamp.value, moment.ISO_8601, true).isValid()) { + callback(new errors.BadTimestamp(timestamp.value, entityName, typeInformation)); + return; + } + } else { + if (!typeInformation.timezone) { + timestamp.value = new Date().toISOString(); + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + } else { + timestamp.value = moment().tz(typeInformation.timezone).format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + } + } + + // Add TimeInstant to all attribute metadata of all entities + for (let ename in entities) { + for (let etype in entities[ename]) { + for (let currentAttr of entities[ename][etype]) { + if (currentAttr.name !== constants.TIMESTAMP_ATTRIBUTE) { + if (!currentAttr.metadata) { + currentAttr.metadata = {}; + } + currentAttr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; + } } - currentMeasure.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; } - //If just measures in the principal entity we missed the Timestamp. } - entities[entityName][typeInformation.type] = entities[entityName][typeInformation.type].concat(measures); } //PRE-PROCESSING FINISHED @@ -587,7 +563,9 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call (item) => attr.object_id !== undefined && item.object_id === attr.object_id )))) ) { - isEmpty = false; + if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { + isEmpty = false; + } e[attr.name] = { type: attr.type, value: attr.value, metadata: attr.metadata }; } } diff --git a/test/functional/testCases.js b/test/functional/testCases.js index d87005902..1d876a783 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2007,30 +2007,6 @@ const testCases = [ } }, should: [ - { - shouldName: - 'A - WHEN sending a measure through http IT should map the measure to timestamp attribute and use it for timestmap sent to Context Broker', - type: 'single', - measure: { - url: 'http://localhost:' + config.http.port + '/iot/json', - method: 'POST', - qs: { - i: globalEnv.deviceId, - k: globalEnv.apikey - }, - json: { - mydatetime: '2022-02-02T02:22:22.222Z' - } - }, - expectation: { - id: globalEnv.entity_name, - type: globalEnv.entity_type, - TimeInstant: { - value: '2022-02-02T02:22:22.222Z', - type: 'DateTime' - } - } - }, { shouldName: 'A - WHEN sending a measure through http IT should map the measure to timestamp attribute and use it for timestmap and other metadata attributes sent to Context Broker', @@ -2175,30 +2151,6 @@ const testCases = [ } }, should: [ - { - shouldName: - 'A - WHEN sending a measure through http IT should map the measure to timestamp attribute and use it for timestmap sent to Context Broker', - type: 'single', - measure: { - url: 'http://localhost:' + config.http.port + '/iot/json', - method: 'POST', - qs: { - i: globalEnv.deviceId, - k: globalEnv.apikey - }, - json: { - mydatetime: '2022-02-02T02:22:22.222Z' - } - }, - expectation: { - id: globalEnv.entity_name, - type: globalEnv.entity_type, - TimeInstant: { - value: '2022-02-02T02:22:22.222Z', - type: 'DateTime' - } - } - }, { shouldName: 'A - WHEN sending a measure through http IT should map the measure to timestamp attribute and use it for timestmap and other metadata attributes sent to Context Broker', @@ -2350,12 +2302,23 @@ const testCases = [ k: globalEnv.apikey }, json: { - mydatetime: '2022-02-02T02:22:22.222Z' + mydatetime: '2022-02-02T02:22:22.222Z', + a: 23 } }, expectation: { id: globalEnv.entity_name, type: globalEnv.entity_type, + a: { + value: 23, + type: 'Text', + metadata: { + TimeInstant: { + value: '2033-03-03T03:33:33.333Z', + type: 'DateTime' + } + } + }, TimeInstant: { value: '2033-03-03T03:33:33.333Z', type: 'DateTime' From 75fe27697390ad4bd14e9ed9c43d8e9d6b7baa70 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 8 Apr 2024 16:17:39 +0200 Subject: [PATCH 11/83] move timestampAttrs to local block --- lib/services/ngsi/entities-NGSI-v2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index bc4d5710d..d6342c8ed 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -278,7 +278,6 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call let jexlctxt = {}; //will store the whole context (not just for JEXL) let payload = {}; //will store the final payload let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. - let timestampAttrs = null; //list of mapped TimeInstant attributes let plainMeasures = null; //will contain measures POJO let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); @@ -498,6 +497,7 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call if (mustInsertTimeInstant) { // search timestamp just in entity attrs + let timestampAttrs = null; //list of mapped TimeInstant attributes for (let ename in entities) { for (let etype in entities[ename]) { timestampAttrs = entities[ename][etype].filter((item) => item.name === constants.TIMESTAMP_ATTRIBUTE); From e940fae4960fda2a789183c31f6b49ad5c268d07 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 9 Apr 2024 13:17:57 +0200 Subject: [PATCH 12/83] add multientity cases --- lib/services/ngsi/entities-NGSI-v2.js | 81 +++++----- test/functional/testCases.js | 141 ++++++++++++++++++ .../ngsiv2/plugins/multientity-plugin_test.js | 36 ++++- 3 files changed, 211 insertions(+), 47 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index d6342c8ed..8b2dfe007 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -277,7 +277,6 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call let entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entoties data striucture let jexlctxt = {}; //will store the whole context (not just for JEXL) let payload = {}; //will store the final payload - let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. let plainMeasures = null; //will contain measures POJO let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); @@ -312,13 +311,12 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call logger.debug( context, - 'sendUpdateValueNgsi2 called with: entityName=%s, measures=%j, typeInformation=%j, initial jexlContext=%j, timestamp=%j with value=%j', + 'sendUpdateValueNgsi2 called with: entityName=%s, measures=%j, typeInformation=%j, initial jexlContext=%j, timestamp=%j', entityName, plainMeasures, typeInformation, jexlctxt, - mustInsertTimeInstant, - timestamp.value + mustInsertTimeInstant ); //Now we can calculate the EntityName of primary entity @@ -495,47 +493,6 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call entities[entityName][typeInformation.type] = entities[entityName][typeInformation.type].concat(measures); } - if (mustInsertTimeInstant) { - // search timestamp just in entity attrs - let timestampAttrs = null; //list of mapped TimeInstant attributes - for (let ename in entities) { - for (let etype in entities[ename]) { - timestampAttrs = entities[ename][etype].filter((item) => item.name === constants.TIMESTAMP_ATTRIBUTE); - if (timestampAttrs && timestampAttrs.length > 0) { - timestamp.value = timestampAttrs[0]['value']; - } - } - } - if (timestamp.value) { - if (!moment(timestamp.value, moment.ISO_8601, true).isValid()) { - callback(new errors.BadTimestamp(timestamp.value, entityName, typeInformation)); - return; - } - } else { - if (!typeInformation.timezone) { - timestamp.value = new Date().toISOString(); - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; - } else { - timestamp.value = moment().tz(typeInformation.timezone).format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; - } - } - - // Add TimeInstant to all attribute metadata of all entities - for (let ename in entities) { - for (let etype in entities[ename]) { - for (let currentAttr of entities[ename][etype]) { - if (currentAttr.name !== constants.TIMESTAMP_ATTRIBUTE) { - if (!currentAttr.metadata) { - currentAttr.metadata = {}; - } - currentAttr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; - } - } - } - } - } - //PRE-PROCESSING FINISHED //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload @@ -548,6 +505,31 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call let e = {}; e.id = String(ename); e.type = String(etype); + let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. + let timestampAttrs = null; + if (mustInsertTimeInstant) { + // get timestamp for current entity + + timestampAttrs = entities[ename][etype].filter((item) => item.name === constants.TIMESTAMP_ATTRIBUTE); + if (timestampAttrs && timestampAttrs.length > 0) { + timestamp.value = timestampAttrs[0]['value']; + } + + if (timestamp.value) { + if (!moment(timestamp.value, moment.ISO_8601, true).isValid()) { + callback(new errors.BadTimestamp(timestamp.value, entityName, typeInformation)); + return; + } + } else { + if (!typeInformation.timezone) { + timestamp.value = new Date().toISOString(); + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; // nosense + } else { + timestamp.value = moment().tz(typeInformation.timezone).format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; // nosense + } + } + } //extract attributes let isEmpty = true; for (let attr of entities[ename][etype]) { @@ -566,6 +548,15 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { isEmpty = false; } + if (mustInsertTimeInstant) { + // Add TimeInstant to all attribute metadata of all entities + if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { + if (!attr.metadata) { + attr.metadata = {}; + } + attr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; + } + } e[attr.name] = { type: attr.type, value: attr.value, metadata: attr.metadata }; } } diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 1d876a783..5105be2d0 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2327,6 +2327,147 @@ const testCases = [ } ] }, + { + describeName: '0433 - Simple group with active attribute + several timestamp mappings defined', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + timestamp: true, + entity_type: globalEnv.entity_type, + commands: [], + lazy: [], + attributes: [ + { + object_id: 'a', + name: 'a', + type: 'Text' + }, + { + object_id: 'mydatetime1', + name: 'TimeInstant', + type: 'DateTime', + entity_name: 'TestType:TestDevice1', + entity_type: 'TestType' + }, + { + object_id: 'mydatetime2', + name: 'TimeInstant', + type: 'DateTime', + entity_name: 'TestType:TestDevice2', + entity_type: 'TestType' + }, + { + object_id: 'a1', + name: 'a1', + type: 'Text', + expression: 'a', + entity_name: 'TestType:TestDevice1', + entity_type: 'TestType' + }, + { + object_id: 'a2', + name: 'a2', + type: 'Text', + expression: 'a', + entity_name: 'TestType:TestDevice2', + entity_type: 'TestType' + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending a measure through http IT should map the measure to timestamp attributes and use it for timestmap and other metadata attributes sent to Context Broker', + type: 'multientity', + isRegex: true, + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 23, + mydatetime1: '2011-01-01T01:11:11.111Z', + mydatetime2: '2022-02-02T02:22:22.222Z' + } + }, + expectation: { + actionType: 'append', + entities: [ + { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + a: { + value: 23, + type: 'Text', + metadata: { + TimeInstant: { + value: _.isDateString, + type: 'DateTime' + } + } + }, + TimeInstant: { + value: _.isDateString, + type: 'DateTime' + } + }, + { + id: 'TestType:TestDevice1', + type: globalEnv.entity_type, + a1: { + value: 23, + type: 'Text', + metadata: { + TimeInstant: { + value: '2011-01-01T01:11:11.111Z', + type: 'DateTime' + } + } + }, + TimeInstant: { + value: '2011-01-01T01:11:11.111Z', + type: 'DateTime' + } + }, + { + id: 'TestType:TestDevice2', + type: globalEnv.entity_type, + a2: { + value: 23, + type: 'Text', + metadata: { + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + }, + TimeInstant: { + value: '2022-02-02T02:22:22.222Z', + type: 'DateTime' + } + } + ] + } + } + ] + }, + // 0500 - EXPLICIT ATTRIBUTES TESTS { describeName: '0500 - Group with explicit attrs:false (boolean) + active atributes', diff --git a/test/unit/ngsiv2/plugins/multientity-plugin_test.js b/test/unit/ngsiv2/plugins/multientity-plugin_test.js index 9e23f6fe6..a1c716e96 100644 --- a/test/unit/ngsiv2/plugins/multientity-plugin_test.js +++ b/test/unit/ngsiv2/plugins/multientity-plugin_test.js @@ -307,6 +307,38 @@ const iotAgentConfig = { } ] }, + WeatherStation10: { + commands: [], + type: 'WeatherStation', + lazy: [], + active: [ + { + object_id: 'p', + name: 'pressure', + type: 'Hgmm' + }, + { + object_id: 'h', + name: 'humidity', + type: 'Percentage', + entity_name: 'Higro2000', + entity_type: 'Higrometer', + metadata: { + unitCode: { + type: 'Text', + value: 'Hgmm' + } + } + }, + { + object_id: 'TimeInstant', + name: 'TimeInstant', + type: 'DateTime', + entity_name: 'Higro2000', + entity_type: 'Higrometer' + } + ] + }, Sensor001: { commands: [], type: 'Sensor', @@ -1488,7 +1520,7 @@ describe('NGSI-v2 - Multi-entity plugin', function () { describe('NGSI-v2 - Multi-entity plugin is executed before timestamp process plugin', function () { beforeEach(function (done) { - logger.setLevel('FATAL'); + logger.setLevel('DEBUG'); iotAgentConfig.timestamp = true; iotAgentLib.activate(iotAgentConfig, function () { iotAgentLib.clearAll(function () { @@ -1635,7 +1667,7 @@ describe('NGSI-v2 - Multi-entity plugin is executed before timestamp process plu value: '2018-06-13T13:28:34.611Z' } ]; - iotAgentLib.update('ws5', 'WeatherStation', '', tsValue, function (error) { + iotAgentLib.update('ws5', 'WeatherStation10', '', tsValue, function (error) { should.not.exist(error); contextBrokerMock.done(); done(); From b5143dcf9fb8216c9c4f1ac47a1369375eb791c1 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 9 Apr 2024 14:35:19 +0200 Subject: [PATCH 13/83] remove unnecessary check --- lib/services/ngsi/entities-NGSI-v2.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 8b2dfe007..7d89db6f4 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -545,9 +545,7 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call (item) => attr.object_id !== undefined && item.object_id === attr.object_id )))) ) { - if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { - isEmpty = false; - } + isEmpty = false; if (mustInsertTimeInstant) { // Add TimeInstant to all attribute metadata of all entities if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { From e921a3c4796642a30acbf8438f9445b9852936ce Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 9 Apr 2024 16:08:30 +0200 Subject: [PATCH 14/83] add metadata in static_attributes case --- test/functional/testCases.js | 69 +++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 5d558c7c3..54298df71 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -1036,7 +1036,8 @@ const testCases = [ ] }, { - describeName: '0140 - Simple group with active attribute + chained JEXL expression text (a | trim | replacestr("hello","hi"))', + describeName: + '0140 - Simple group with active attribute + chained JEXL expression text (a | trim | replacestr("hello","hi"))', provision: { url: 'http://localhost:' + config.iota.server.port + '/iot/services', method: 'POST', @@ -1704,6 +1705,72 @@ const testCases = [ } ] }, + { + describeName: '0320 - Simple group with active attributes + metadata in Static attributes ', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + commands: [], + lazy: [], + attributes: [], + static_attributes: [ + { + name: 'static_a', + type: 'Number', + value: 4, + metadata: { + color: 'blue' + } + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending measures through http IT should store metatada static attributes into Context Broker', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 10 + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + a: { + value: 10, + type: 'Text' + }, + static_a: { + value: 4, + type: 'Number', + metadata: { + color: 'blue' + } + } + } + } + ] + }, // 0400 - TIMESTAMP TESTS { describeName: '0400 - Simple group with active attribute + timestamp:false', From cec0319d21744e42dc7eb171ac9c3c6a9238a12c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 9 Apr 2024 16:48:11 +0200 Subject: [PATCH 15/83] update to use typical metadata example --- test/functional/testCases.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 54298df71..ff6d878ba 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -1725,7 +1725,10 @@ const testCases = [ type: 'Number', value: 4, metadata: { - color: 'blue' + accuracy: { + value: 0.8, + type: 'Float' + } } } ] @@ -1764,7 +1767,10 @@ const testCases = [ value: 4, type: 'Number', metadata: { - color: 'blue' + accuracy: { + value: 0.8, + type: 'Float' + } } } } From ce0bc8637f826c1684613a5952e710a554debe1f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 9 Apr 2024 17:07:34 +0200 Subject: [PATCH 16/83] add metadata to mapping attribute case --- test/functional/testCases.js | 68 ++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index ff6d878ba..242191f0e 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -445,6 +445,74 @@ const testCases = [ } ] }, + { + describeName: '0021 Simple group with active attributes with metadata', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + commands: [], + lazy: [], + attributes: [ + { + object_id: 'a', + name: 'attr_a', + type: 'Boolean', + metadata: { + accuracy: { + value: 0.8, + type: 'Float' + } + } + } + ], + static_attributes: [] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending defined object_ids (measures) through http IT should send measures to Context Broker preserving value types, name mappings and metadatas', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: false + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + attr_a: { + value: false, + type: 'Boolean', + metadata: { + accuracy: { + value: 0.8, + type: 'Float' + } + } + } + } + } + ] + }, // 0100 - JEXL TESTS { describeName: '0100 - Simple group with active attribute + JEXL expression boolean (!)', From ca5cd359ebcd7de03abc5902f7158b0fa9337099 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 10 Apr 2024 09:41:01 +0200 Subject: [PATCH 17/83] add tests about empty multientity mapped timestamp entites --- test/functional/testCases.js | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 5105be2d0..3d85e08e3 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2328,7 +2328,7 @@ const testCases = [ ] }, { - describeName: '0433 - Simple group with active attribute + several timestamp mappings defined', + describeName: '0433 - Simple group with active attribute + several timestamp mappings defined in multientity', provision: { url: 'http://localhost:' + config.iota.server.port + '/iot/services', method: 'POST', @@ -2464,6 +2464,36 @@ const testCases = [ } ] } + }, + { + shouldName: + 'A - WHEN sending a measure through http IT should map the measure to timestamp attributes for just mapped entity and sent to Context Broker', + type: 'multientity', + isRegex: true, + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + mydatetime1: '2011-01-01T01:11:11.111Z' + } + }, + expectation: { + actionType: 'append', + entities: [ + { + id: 'TestType:TestDevice1', + type: globalEnv.entity_type, + TimeInstant: { + value: '2011-01-01T01:11:11.111Z', + type: 'DateTime' + } + } + ] + } } ] }, From db6b7d7c4e2c380cf5793b8f9685e9f90e3a6112 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 10 Apr 2024 15:19:17 +0200 Subject: [PATCH 18/83] process metadata expression --- lib/services/ngsi/entities-NGSI-v2.js | 40 ++++++++++++++-- test/functional/testCases.js | 69 +++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 38d0a1471..7bf95e29f 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -308,10 +308,7 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call jexlctxt = reduceAttrToPlainObject(idTypeSSSList, jexlctxt); //Managing timestamp (mustInsertTimeInstant flag to decide if we should insert Timestamp later on) - const mustInsertTimeInstant = - typeInformation.timestamp !== undefined - ? typeInformation.timestamp - : false; + const mustInsertTimeInstant = typeInformation.timestamp !== undefined ? typeInformation.timestamp : false; if (mustInsertTimeInstant) { //remove TimeInstant from measures @@ -486,7 +483,6 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call } currentAttr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; } - //store de New Attributte in entity data structure if (hitted === true) { if (entities[attrEntityName] === undefined) { @@ -501,6 +497,40 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call //RE-Populate de JEXLcontext (except for null or NaN we preffer undefined) jexlctxt[currentAttr.name] = valueExpression; + + // Expand metadata value expression + if (currentAttr.metadata) { + for (var metaKey in currentAttr.metadata) { + if (!currentAttr.metadata.hasOwnProperty(metaKey)) continue; + + if (currentAttr.metadata[metaKey].expression && metaKey !== constants.TIMESTAMP_ATTRIBUTE) { + let newAttrMeta = {}; + if (currentAttr.metadata[metaKey].type) { + newAttrMeta['type'] = currentAttr.metadata[metaKey].type; + } + let metaValueExpression; + try { + metaValueExpression = jexlParser.applyExpression( + currentAttr.metadata[metaKey].expression, + jexlctxt, + typeInformation + ); + //we fallback to null if anything unexpecte happend + if ( + metaValueExpression === null || + metaValueExpression === undefined || + Number.isNaN(metaValueExpression) + ) { + metaValueExpression = null; + } + } catch (e) { + metaValueExpression = null; + } + newAttrMeta['value'] = metaValueExpression; + currentAttr.metadata[metaKey] = newAttrMeta; + } + } + } } //now we can compute explicit (Bool or Array) with the complete JexlContext diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 242191f0e..3c90276b6 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -1545,6 +1545,75 @@ const testCases = [ } ] }, + { + describeName: '0191 - Simple group with JEXL expression in metadata', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + commands: [], + lazy: [], + attributes: [ + { + object_id: 'a', + name: 'attr_a', + type: 'Boolean', + expression: 'a?threshold[90|tostring].max:true', + metadata: { + unit: { + type: 'Text', + expression: '"hola" + "adios"' + } + } + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending a measure through http IT should send to Context Broker expanded metadata value ', + type: 'single', + isRegex: true, + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: false + } + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + attr_a: { + value: true, + type: 'Boolean', + metadata: { + unit: { + type: 'Text', + value: 'holaadios' + } + } + } + } + } + ] + }, // 0200 - COMMANDS TESTS { describeName: '0200 - Simple group with commands', From b75f2779dd44c87b402a7c7d872d7d74a56b36ad Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 10 Apr 2024 15:27:59 +0200 Subject: [PATCH 19/83] fix linter --- CHANGES_NEXT_RELEASE | 1 + lib/services/ngsi/entities-NGSI-v2.js | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e03e13f6c..f057d8347 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ +- Add: process JEXL expressions in metadata attributes - Fix: reduce information showed handling errors to just config flags (#1594) - Upgrade express dep from 4.18.1 to 4.19.2 - Add: allow devices with the same device_id in the same service and subservice but different apikey (#1589) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 7bf95e29f..d6b0e4e38 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -501,8 +501,6 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call // Expand metadata value expression if (currentAttr.metadata) { for (var metaKey in currentAttr.metadata) { - if (!currentAttr.metadata.hasOwnProperty(metaKey)) continue; - if (currentAttr.metadata[metaKey].expression && metaKey !== constants.TIMESTAMP_ATTRIBUTE) { let newAttrMeta = {}; if (currentAttr.metadata[metaKey].type) { From 7896b7f16a81101a7e85153e73e7af5b1971c6da Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 10 Apr 2024 15:29:18 +0200 Subject: [PATCH 20/83] update CNR --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index f057d8347..041ddc8dd 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Add: process JEXL expressions in metadata attributes +- Add: process JEXL expressions in metadata attributes (#1601) - Fix: reduce information showed handling errors to just config flags (#1594) - Upgrade express dep from 4.18.1 to 4.19.2 - Add: allow devices with the same device_id in the same service and subservice but different apikey (#1589) From 4c8a62478a645ff66218e73012b3b52c0c9da833 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 10 Apr 2024 15:46:20 +0200 Subject: [PATCH 21/83] pre-calculate currentIsoDate --- lib/services/ngsi/entities-NGSI-v2.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 7d89db6f4..b888789ab 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -500,6 +500,8 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call payload.actionType = 'append'; payload.entities = []; + const currentIsoDate = new Date().toISOString(); + const currentMoment = moment(); for (let ename in entities) { for (let etype in entities[ename]) { let e = {}; @@ -522,11 +524,13 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call } } else { if (!typeInformation.timezone) { - timestamp.value = new Date().toISOString(); - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; // nosense + timestamp.value = currentIsoDate; + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; } else { - timestamp.value = moment().tz(typeInformation.timezone).format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; // nosense + timestamp.value = currentMoment + .tz(typeInformation.timezone) + .format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; } } } From 787dc5f3a689c4d460ab2c17c6c2398ddc11928e Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 16 Apr 2024 15:25:42 +0200 Subject: [PATCH 22/83] update doc --- doc/api.md | 45 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/doc/api.md b/doc/api.md index f143f8805..33e4815b8 100644 --- a/doc/api.md +++ b/doc/api.md @@ -157,8 +157,8 @@ parameters defined at device level in database, the parameters are inherit from Group service uniqueness is defined by the combination of: service, subservice and apikey -Device uniqueness is defined by the combination of: service, subservice, device_id and apikey. Note that several -devices with the same device_id are allowed in the same service and subservice as long as their apikeys are different. +Device uniqueness is defined by the combination of: service, subservice, device_id and apikey. Note that several devices +with the same device_id are allowed in the same service and subservice as long as their apikeys are different. ## Special measures and attributes names @@ -314,6 +314,43 @@ e.g.: } ``` +Metadata could also has `expression` like attributes in order to expand it: + +e.g.: + +```json +{ + "entity_type": "Lamp", + "resource": "/iot/d", + "protocol": "PDI-IoTA-UltraLight", +..etc + "commands": [ + {"name": "on","type": "command"}, + {"name": "off","type": "command"} + ], + "attributes": [ + {"object_id": "s", "name": "state", "type":"Text"}, + {"object_id": "l", "name": "luminosity", "type":"Integer", + "metadata":{ + "unitCode":{"type": "Text", "value" :"CAL"} + } + } + ], + "static_attributes": [ + {"name": "category", "type":"Text", "value": ["actuator","sensor"]}, + {"name": "controlledProperty", "type": "Text", "value": ["light"], + "metadata":{ + "includes":{"type": "Text", + "value" :["state", "luminosity"], + "expression": "level / 100" + }, + "alias":{"type": "Text", "value" :"lamp"} + } + }, + ] + } +``` + ### NGSI-LD data and metadata considerations When provisioning devices for an NGSI-LD Context Broker, `type` values should typically correspond to one of the @@ -525,8 +562,8 @@ expression. In all cases the following data is available to all expressions: - `subservice`: device subservice - `staticAttributes`: static attributes defined in the device or config group -Additionally, for attribute expressions (`expression`, `entity_name`) and `entityNameExp` measures are avaiable in the -**context** used to evaluate them. +Additionally, for attribute expressions (`expression`, `entity_name`) , `entityNameExp` andmetadata expressions +(`expression`) measures are avaiable in the **context** used to evaluate them. ### Examples of JEXL expressions From abe5cbb67877c92b96d91cf8dea3b42abc14ea1a Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 16 Apr 2024 15:43:56 +0200 Subject: [PATCH 23/83] update doc --- doc/api.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/doc/api.md b/doc/api.md index 2065f8274..07603719a 100644 --- a/doc/api.md +++ b/doc/api.md @@ -977,19 +977,23 @@ adds a `TimeInstant` attribute as metadata for every other attribute in the same `observedAt` property-of-a-property is used instead. If a `TimeInstant` arrives as measure but not follows [ISO_8601](https://en.wikipedia.org/wiki/ISO_8601) then measure is -refused. +refused. In this case `TimeInstant` could be mapped to an attribute and then modified by an expression, see the +[Expression Language definition](#expression-language-support) for details Depending on the `timestamp` configuration and if the measure contains a value named `TimeInstant` with a correct value, the IoTA behaviour is described in the following table: -| `timestamp` value | measure contains `TimeInstant` | Behaviour | -| ----------------- | ------------------------------ | ------------------------------------------------------ | -| true | Yes | TimeInstant and metadata updated with measure value | -| true | No | TimeInstant and metadata updated with server timestamp | -| false | Yes | TimeInstant and metadata updated with measure value | -| false | No | TimeInstant and metadata updated with server timestamp | -| Not defined | Yes | TimeInstant and metadata updated with measure value | -| Not defined | No | TimeInstant and metadata updated with server timestamp | +| `timestamp` conf value | measure contains `TimeInstant` | Behaviour | +| ---------------------- | ------------------------------ | ------------------------------------------------------ | +| true | Yes | TimeInstant and metadata updated with measure value | +| true | No | TimeInstant and metadata updated with server timestamp | +| false | Yes | TimeInstant and metadata updated with measure value | +| false | No | TimeInstant and metadata updated with server timestamp | +| Not defined | Yes | TimeInstant and metadata updated with measure value | +| Not defined | No | TimeInstant and metadata updated with server timestamp | + +If there is an attribute with maps a measure to a TimeInstant, then that value will be used as a default TimeInstant, +overwriting the above rules, that is, a `TimeInstant` measure or server timestamp. The `timestamp` value used is: From 5e0840a1ac43f5377d4b7429ce40ddb15aa7df56 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 16 Apr 2024 17:31:37 +0200 Subject: [PATCH 24/83] reuse currentIsoDate in moment --- lib/services/ngsi/entities-NGSI-v2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index b888789ab..6cb2ec9f6 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -501,7 +501,7 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call payload.entities = []; const currentIsoDate = new Date().toISOString(); - const currentMoment = moment(); + const currentMoment = moment(currentIsoDate); for (let ename in entities) { for (let etype in entities[ename]) { let e = {}; From 8f40178d48d26fe1c34c55f0d61e9ff39e20ce33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 17 Apr 2024 17:03:29 +0200 Subject: [PATCH 25/83] FIX documentation and changelog --- CHANGES_NEXT_RELEASE | 2 +- doc/api.md | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0d5821587..a36c11521 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Fix timestamp attribute mapped cases (#1557) +- Fix: TimeInstant mapped from measure overrides system TimeInstant (#1557) - Fix: reduce information showed handling errors to just config flags (#1594) - Upgrade express dep from 4.18.1 to 4.19.2 - Add: allow devices with the same device_id in the same service and subservice but different apikey (#1589) diff --git a/doc/api.md b/doc/api.md index 07603719a..32e8db007 100644 --- a/doc/api.md +++ b/doc/api.md @@ -972,13 +972,11 @@ Will now generate the following NGSI v2 payload: ## Timestamp Processing -The IOTA processes the entity attributes looking for a `TimeInstant` attribute. If one is found, for NGSI v2, then it -adds a `TimeInstant` attribute as metadata for every other attribute in the same request. With NGSI-LD, the Standard -`observedAt` property-of-a-property is used instead. +Timestamp processing done by IOTA is as follows: -If a `TimeInstant` arrives as measure but not follows [ISO_8601](https://en.wikipedia.org/wiki/ISO_8601) then measure is -refused. In this case `TimeInstant` could be mapped to an attribute and then modified by an expression, see the -[Expression Language definition](#expression-language-support) for details +* An attribute `TimeInstant` is added to updated entities +* In the case of NGSI-v2, a `TimeInstant` metadata is added in each updated attribute. With NGSI-LD, the Standard +`observedAt` property-of-a-property is used instead. Depending on the `timestamp` configuration and if the measure contains a value named `TimeInstant` with a correct value, the IoTA behaviour is described in the following table: @@ -992,16 +990,19 @@ the IoTA behaviour is described in the following table: | Not defined | Yes | TimeInstant and metadata updated with measure value | | Not defined | No | TimeInstant and metadata updated with server timestamp | -If there is an attribute with maps a measure to a TimeInstant, then that value will be used as a default TimeInstant, -overwriting the above rules, that is, a `TimeInstant` measure or server timestamp. - -The `timestamp` value used is: +The `timestamp` conf value used is: - The one defined at device level - The one defined at group level (if not defined at device level) - The one defined at [IoTA configuration level](admin.md#timestamp) / `IOTA_TIMESTAMP` env var (if not defined at group level or device level) +Some additional considerations to take into account: + +* If there is an attribute which maps a measure to `TimeInstant` attribute (after [expression evaluation](#expression-language-support) if any is defined), then that value will be used as as `TimeInstant, +overwriting the above rules specified in "Behaviour" column. Note that an expression in the could be used in that mapping. +* If the resulting `TimeInstant` not follows [ISO_8601](https://en.wikipedia.org/wiki/ISO_8601) (either from a direct measure of after a mapping, as described in the previous bullet) then it is refused (so a failover to server timestamp will take place). + ## Overriding global Context Broker host **cbHost**: Context Broker host URL. This option can be used to override the global CB configuration for specific types From 77acdd68c782358cb648938368d41013363dc1ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 17 Apr 2024 17:07:18 +0200 Subject: [PATCH 26/83] Update doc/api.md --- doc/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api.md b/doc/api.md index 32e8db007..320bf240b 100644 --- a/doc/api.md +++ b/doc/api.md @@ -999,7 +999,7 @@ The `timestamp` conf value used is: Some additional considerations to take into account: -* If there is an attribute which maps a measure to `TimeInstant` attribute (after [expression evaluation](#expression-language-support) if any is defined), then that value will be used as as `TimeInstant, +* If there is an attribute which maps a measure to `TimeInstant` attribute (after [expression evaluation](#expression-language-support) if any is defined), then that value will be used as `TimeInstant, overwriting the above rules specified in "Behaviour" column. Note that an expression in the could be used in that mapping. * If the resulting `TimeInstant` not follows [ISO_8601](https://en.wikipedia.org/wiki/ISO_8601) (either from a direct measure of after a mapping, as described in the previous bullet) then it is refused (so a failover to server timestamp will take place). From 9d90e72b1fc24d1edc15a6f2a031cd26606a80b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 17 Apr 2024 17:42:48 +0200 Subject: [PATCH 27/83] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index a36c11521..903e94d8a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Fix: TimeInstant mapped from measure overrides system TimeInstant (#1557) +- Fix: TimeInstant mapped from attribute overrides default behaviours (#1557) - Fix: reduce information showed handling errors to just config flags (#1594) - Upgrade express dep from 4.18.1 to 4.19.2 - Add: allow devices with the same device_id in the same service and subservice but different apikey (#1589) From 20196fff8ba8876887a0d15f4cb726f33f8d35d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 18 Apr 2024 09:44:43 +0200 Subject: [PATCH 28/83] Update doc/api.md --- doc/api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/api.md b/doc/api.md index 13795cd5c..d661861c0 100644 --- a/doc/api.md +++ b/doc/api.md @@ -562,8 +562,8 @@ expression. In all cases the following data is available to all expressions: - `subservice`: device subservice - `staticAttributes`: static attributes defined in the device or config group -Additionally, for attribute expressions (`expression`, `entity_name`) , `entityNameExp` andmetadata expressions -(`expression`) measures are avaiable in the **context** used to evaluate them. +Additionally, for attribute expressions (`expression`, `entity_name`), `entityNameExp` and metadata expressions +(`expression`) measures are available in the **context** used to evaluate them. ### Examples of JEXL expressions From 7310b7970193fb174c44779facaa6f496b181834 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 19 Apr 2024 14:22:02 +0200 Subject: [PATCH 29/83] FIX reference in CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 29949b3a4..ddbd2cd7f 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Add: process JEXL expressions in metadata attributes (#1601) +- Add: process JEXL expressions in metadata attributes (#1598) - Fix: TimeInstant mapped from attribute overrides default behaviours (#1557) - Fix: reduce information showed handling errors to just config flags (#1594) - Upgrade pymongo dep from 4.3.3 to 4.6.3 From 8c99cf39f9c0315217567336cfdee2ad27e6560f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 26 Apr 2024 12:01:04 +0200 Subject: [PATCH 30/83] step 4.4.0-next -> 4.4.0 --- CHANGES_NEXT_RELEASE | 7 ------- package.json | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index ddbd2cd7f..e69de29bb 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,7 +0,0 @@ -- Add: process JEXL expressions in metadata attributes (#1598) -- Fix: TimeInstant mapped from attribute overrides default behaviours (#1557) -- Fix: reduce information showed handling errors to just config flags (#1594) -- Upgrade pymongo dep from 4.3.3 to 4.6.3 -- Upgrade express dep from 4.18.1 to 4.19.2 -- Add: allow devices with the same device_id in the same service and subservice but different apikey (#1589) - diff --git a/package.json b/package.json index ac6d5c663..96db30177 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "iotagent-node-lib", "license": "AGPL-3.0-only", "description": "IoT Agent library to interface with NGSI Context Broker", - "version": "4.3.0-next", + "version": "4.4.0", "homepage": "https://github.com/telefonicaid/iotagent-node-lib", "keywords": [ "fiware", From 6750d39ad51ca98e964981069ccf22428878dc36 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 26 Apr 2024 13:09:43 +0200 Subject: [PATCH 31/83] step 4.4.0 -> 4.4.0-next --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 96db30177..c9fc57981 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "iotagent-node-lib", "license": "AGPL-3.0-only", "description": "IoT Agent library to interface with NGSI Context Broker", - "version": "4.4.0", + "version": "4.4.0-next", "homepage": "https://github.com/telefonicaid/iotagent-node-lib", "keywords": [ "fiware", From 6fda88bd441bd1f45a35ef238f94a8df4db9b8a9 Mon Sep 17 00:00:00 2001 From: mapedraza Date: Tue, 30 Apr 2024 12:10:57 +0200 Subject: [PATCH 32/83] add tests failing --- test/functional/testCases.js | 259 +++++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 9f88f75d1..2232d2a10 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2968,6 +2968,265 @@ const testCases = [ } ] }, + { + describeName: '0531 - Group with explicit attrs:[ ] (empty array) + active attributes + static attributes', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + explicitAttrs: '[ ]', + commands: [], + lazy: [], + attributes: [ + { + name: 'attr_a', + object_id: 'a', + type: 'Number' + }, + { + name: 'attr_b', + object_id: 'b', + type: 'Number' + } + ], + static_attributes: [ + { + name: 'static_a', + type: 'Number', + value: 3 + }, + { + name: 'static_b', + type: 'Number', + value: 4 + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending both provisioned and not object_ids (measures) through http IT should NOT store anything into the Context Broker (No request to CB)', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 3, + b: 10, + c: 11 + } + }, + expectation: [] // No payload expected + } + ] + }, + { + describeName: + '0532 - Group with explicit attrs:[ ] (empty array) + active attributes + static attributes + timestamp:true', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + explicitAttrs: '[ ]', + timestamp: true, + commands: [], + lazy: [], + attributes: [ + { + name: 'attr_a', + object_id: 'a', + type: 'Number' + }, + { + name: 'attr_b', + object_id: 'b', + type: 'Number' + } + ], + static_attributes: [ + { + name: 'static_a', + type: 'Number', + value: 3 + }, + { + name: 'static_b', + type: 'Number', + value: 4 + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending both provisioned and not object_ids (measures) through http IT should NOT store anything into the Context Broker (No request to CB)', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 3, + b: 10, + c: 11 + } + }, + expectation: [] // No payload expected + }, + { + shouldName: + 'B - WHEN sending data and a measure called TimeInstant through http IT should NOT store anything into the Context Broker (No request to CB)', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 3, + b: 10, + c: 11, + TimeInstant: '2015-12-14T08:06:01.468Z' + } + }, + expectation: [] // No payload expected + } + ] + }, + { + describeName: + '0533 - Group with explicit attrs:[ ] (empty array) + active attributes + TimeInstant attribute + static attributes + timestamp:true', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + explicitAttrs: "['DateIssued']", + timestamp: true, + commands: [], + lazy: [], + attributes: [ + { + name: 'attr_a', + object_id: 'a', + type: 'Number' + }, + { + name: 'attr_b', + object_id: 'b', + type: 'Number' + }, + { + name: 'DateIssued', + object_id: 'TimeInstant', + type: 'DateTime' + }, + { + name: 'TimeInstant', + object_id: 't', + type: 'DateTime' + } + ], + static_attributes: [ + { + name: 'static_a', + type: 'Number', + value: 3 + }, + { + name: 'static_b', + type: 'Number', + value: 4 + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending data and a measure called "t" (defined as TimeInstant attribte) through http IT should NOT store anything into the Context Broker (No request to CB)', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 3, + b: 10, + c: 11, + t: '2015-12-14T08:06:01.468Z' + } + }, + expectation: {} + }, + { + shouldName: + 'B - WHEN sending data and a measure called TimeInstant through http IT should NOT store anything into the Context Broker (No request to CB)', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 3, + b: 10, + c: 11, + TimeInstant: '2015-12-14T08:06:01.468Z' + } + }, + expectation: {} // No payload expected + } + ] + }, { describeName: '0540 - Group with explicit attrs: JEXL expression based on measure resulting boolean + active attributes + static attributes', From 0801e1fc22a5e194c3fdf430a8a10ce1a01b9e65 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 6 May 2024 09:50:24 +0200 Subject: [PATCH 33/83] allow to send to CB batch update for multimeasures --- lib/services/ngsi/entities-NGSI-v2.js | 563 ++++++++++++++------------ 1 file changed, 293 insertions(+), 270 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index dc5e3b733..d8514b6ac 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -260,7 +260,7 @@ function sendQueryValueNgsi2(entityName, attributes, typeInformation, token, cal * @param {Object} typeInformation Configuration information for the device. * @param {String} token User token to identify against the PEP Proxies (optional). */ -function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, callback) { +function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token, callback) { //aux function used to builf JEXL context. //it returns a flat object from an Attr array function reduceAttrToPlainObject(attrs, initObj = {}) { @@ -274,335 +274,354 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call } } - let entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entoties data striucture - let jexlctxt = {}; //will store the whole context (not just for JEXL) - let payload = {}; //will store the final payload - let plainMeasures = null; //will contain measures POJO let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); //Make a clone and overwrite typeInformation = JSON.parse(JSON.stringify(typeInformation)); - //Check mandatory information: type if (!typeInformation || !typeInformation.type) { callback(new errors.TypeNotFound(null, entityName, typeInformation)); return; } - //Rename all measures with matches with id and type to measure_id and measure_type - for (let measure of measures) { - if (measure.name === 'id' || measure.name === 'type') { - measure.name = constants.MEASURE + measure.name; - } - } - - //Make a copy of measures in an plain object: plainMeasures - plainMeasures = reduceAttrToPlainObject(measures); - //Build the initital JEXL Context - //All the measures (avoid references make another copy instead) - jexlctxt = reduceAttrToPlainObject(measures); - //All the static - jexlctxt = reduceAttrToPlainObject(typeInformation.staticAttributes, jexlctxt); - //id type Service and Subservice - jexlctxt = reduceAttrToPlainObject(idTypeSSSList, jexlctxt); + let payload = {}; //will store the final payload + let entities = {}; + payload.actionType = 'append'; + payload.entities = []; + const currentIsoDate = new Date().toISOString(); + const currentMoment = moment(currentIsoDate); //Managing timestamp (mustInsertTimeInstant flag to decide if we should insert Timestamp later on) const mustInsertTimeInstant = typeInformation.timestamp !== undefined ? typeInformation.timestamp : false; - logger.debug( - context, - 'sendUpdateValueNgsi2 called with: entityName=%s, measures=%j, typeInformation=%j, initial jexlContext=%j, timestamp=%j', - entityName, - plainMeasures, - typeInformation, - jexlctxt, - mustInsertTimeInstant - ); + // Check if measures is a single measure or a array of measures (a multimeasure) + if (originMeasures[0] && !originMeasures[0][0]) { + originMeasures = [originMeasures]; + } + for (let measures of originMeasures) { + entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entoties data striucture + let jexlctxt = {}; //will store the whole context (not just for JEXL) - //Now we can calculate the EntityName of primary entity - let entityNameCalc = null; - if (typeInformation.entityNameExp !== undefined && typeInformation.entityNameExp !== '') { - try { - logger.debug(context, 'sendUpdateValueNgsi2 entityNameExp %j', typeInformation.entityNameExp); - entityNameCalc = expressionPlugin.applyExpression(typeInformation.entityNameExp, jexlctxt, typeInformation); - } catch (e) { - logger.debug( - context, - 'Error evaluating expression for entityName: %j with context: %j', - typeInformation.entityNameExp, - jexlctxt - ); + let plainMeasures = null; //will contain measures POJO + + //Rename all measures with matches with id and type to measure_id and measure_type + for (let measure of measures) { + if (measure.name === 'id' || measure.name === 'type') { + measure.name = constants.MEASURE + measure.name; + } } - } - entityName = entityNameCalc ? entityNameCalc : entityName; - //enrich JEXL context - jexlctxt['entity_name'] = entityName; + //Make a copy of measures in an plain object: plainMeasures + plainMeasures = reduceAttrToPlainObject(measures); + //Build the initital JEXL Context + //All the measures (avoid references make another copy instead) + jexlctxt = reduceAttrToPlainObject(measures); + //All the static + jexlctxt = reduceAttrToPlainObject(typeInformation.staticAttributes, jexlctxt); + //id type Service and Subservice + jexlctxt = reduceAttrToPlainObject(idTypeSSSList, jexlctxt); - let preprocessedAttr = []; - //Add Raw Static, Lazy, Command and Actives attr attributes - if (typeInformation && typeInformation.staticAttributes) { - preprocessedAttr = preprocessedAttr.concat(typeInformation.staticAttributes); - } - if (typeInformation && typeInformation.lazy) { - preprocessedAttr = preprocessedAttr.concat(typeInformation.lazy); - } - if (typeInformation && typeInformation.active) { - preprocessedAttr = preprocessedAttr.concat(typeInformation.active); - } + logger.debug( + context, + 'sendUpdateValueNgsi2 loop with: entityName=%s, measures=%j, typeInformation=%j, initial jexlContext=%j, timestamp=%j', + entityName, + plainMeasures, + typeInformation, + jexlctxt, + mustInsertTimeInstant + ); - //Proccess every proto Attribute to populate entities data steuture - entities[entityName] = {}; - entities[entityName][typeInformation.type] = []; - - for (let currentAttr of preprocessedAttr) { - let hitted = false; //any measure, expressiom or value hit the attr (avoid propagate "silent attr" with null values ) - let attrEntityName = entityName; - let attrEntityType = typeInformation.type; - let valueExpression = null; - //manage active attr without object__id (name by default) - currentAttr.object_id = currentAttr.object_id ? currentAttr.object_id : currentAttr.name; - //Enrich the attr (skip, hit, value, meta-timeInstant) - currentAttr.skipValue = currentAttr.skipValue ? currentAttr.skipValue : null; - - //determine AttrEntityName for multientity - if ( - currentAttr.entity_name !== null && - currentAttr.entity_name !== undefined && - currentAttr.entity_name !== '' && - typeof currentAttr.entity_name == 'string' - ) { + //Now we can calculate the EntityName of primary entity + let entityNameCalc = null; + if (typeInformation.entityNameExp !== undefined && typeInformation.entityNameExp !== '') { try { - logger.debug( - context, - 'Evaluating attribute: %j, for entity_name(exp):%j, with ctxt: %j', - currentAttr.name, - currentAttr.entity_name, - jexlctxt + logger.debug(context, 'sendUpdateValueNgsi2 entityNameExp %j', typeInformation.entityNameExp); + entityNameCalc = expressionPlugin.applyExpression( + typeInformation.entityNameExp, + jexlctxt, + typeInformation ); - attrEntityName = jexlParser.applyExpression(currentAttr.entity_name, jexlctxt, typeInformation); - if (!attrEntityName) { - attrEntityName = currentAttr.entity_name; - } } catch (e) { logger.debug( context, - 'Exception evaluating entityNameExp:%j, with jexlctxt: %j', - currentAttr.entity_name, + 'Error evaluating expression for entityName: %j with context: %j', + typeInformation.entityNameExp, jexlctxt ); - attrEntityName = currentAttr.entity_name; } } - //determine AttrEntityType for multientity - if ( - currentAttr.entity_type !== null && - currentAttr.entity_type !== undefined && - currentAttr.entity_type !== '' && - typeof currentAttr.entity_type === 'string' - ) { - attrEntityType = currentAttr.entity_type; - } + entityName = entityNameCalc ? entityNameCalc : entityName; + //enrich JEXL context + jexlctxt['entity_name'] = entityName; - //PRE POPULATE CONTEXT - jexlctxt[currentAttr.name] = plainMeasures[currentAttr.object_id]; - - //determine Value - if (currentAttr.value !== undefined) { - //static attributes already have a value - hitted = true; - valueExpression = currentAttr.value; - } else if (plainMeasures[currentAttr.object_id] !== undefined) { - //we have got a meaure for that Attr - //actives ¿lazis? - hitted = true; - valueExpression = plainMeasures[currentAttr.object_id]; + let preprocessedAttr = []; + //Add Raw Static, Lazy, Command and Actives attr attributes + if (typeInformation && typeInformation.staticAttributes) { + preprocessedAttr = preprocessedAttr.concat(typeInformation.staticAttributes); } - //remove measures that has been shadowed by an alias (some may be left and managed later) - //Maybe we must filter object_id if there is name == object_id - measures = measures.filter((item) => item.name !== currentAttr.object_id && item.name !== currentAttr.name); - - if ( - currentAttr.expression !== undefined && - currentAttr.expression !== '' && - typeof currentAttr.expression == 'string' - ) { - try { + if (typeInformation && typeInformation.lazy) { + preprocessedAttr = preprocessedAttr.concat(typeInformation.lazy); + } + if (typeInformation && typeInformation.active) { + preprocessedAttr = preprocessedAttr.concat(typeInformation.active); + } + + //Proccess every proto Attribute to populate entities data steuture + entities[entityName] = {}; + entities[entityName][typeInformation.type] = []; + + for (let currentAttr of preprocessedAttr) { + let hitted = false; //any measure, expressiom or value hit the attr (avoid propagate "silent attr" with null values ) + let attrEntityName = entityName; + let attrEntityType = typeInformation.type; + let valueExpression = null; + //manage active attr without object__id (name by default) + currentAttr.object_id = currentAttr.object_id ? currentAttr.object_id : currentAttr.name; + //Enrich the attr (skip, hit, value, meta-timeInstant) + currentAttr.skipValue = currentAttr.skipValue ? currentAttr.skipValue : null; + + //determine AttrEntityName for multientity + if ( + currentAttr.entity_name !== null && + currentAttr.entity_name !== undefined && + currentAttr.entity_name !== '' && + typeof currentAttr.entity_name == 'string' + ) { + try { + logger.debug( + context, + 'Evaluating attribute: %j, for entity_name(exp):%j, with ctxt: %j', + currentAttr.name, + currentAttr.entity_name, + jexlctxt + ); + attrEntityName = jexlParser.applyExpression(currentAttr.entity_name, jexlctxt, typeInformation); + if (!attrEntityName) { + attrEntityName = currentAttr.entity_name; + } + } catch (e) { + logger.debug( + context, + 'Exception evaluating entityNameExp:%j, with jexlctxt: %j', + currentAttr.entity_name, + jexlctxt + ); + attrEntityName = currentAttr.entity_name; + } + } + + //determine AttrEntityType for multientity + if ( + currentAttr.entity_type !== null && + currentAttr.entity_type !== undefined && + currentAttr.entity_type !== '' && + typeof currentAttr.entity_type === 'string' + ) { + attrEntityType = currentAttr.entity_type; + } + + //PRE POPULATE CONTEXT + jexlctxt[currentAttr.name] = plainMeasures[currentAttr.object_id]; + + //determine Value + if (currentAttr.value !== undefined) { + //static attributes already have a value + hitted = true; + valueExpression = currentAttr.value; + } else if (plainMeasures[currentAttr.object_id] !== undefined) { + //we have got a meaure for that Attr + //actives ¿lazis? hitted = true; - valueExpression = jexlParser.applyExpression(currentAttr.expression, jexlctxt, typeInformation); - //we fallback to null if anything unexpecte happend - if (valueExpression === null || valueExpression === undefined || Number.isNaN(valueExpression)) { + valueExpression = plainMeasures[currentAttr.object_id]; + } + //remove measures that has been shadowed by an alias (some may be left and managed later) + //Maybe we must filter object_id if there is name == object_id + measures = measures.filter((item) => item.name !== currentAttr.object_id && item.name !== currentAttr.name); + + if ( + currentAttr.expression !== undefined && + currentAttr.expression !== '' && + typeof currentAttr.expression == 'string' + ) { + try { + hitted = true; + valueExpression = jexlParser.applyExpression(currentAttr.expression, jexlctxt, typeInformation); + //we fallback to null if anything unexpecte happend + if (valueExpression === null || valueExpression === undefined || Number.isNaN(valueExpression)) { + valueExpression = null; + } + } catch (e) { valueExpression = null; } - } catch (e) { - valueExpression = null; + logger.debug( + context, + 'Evaluated attr: %j, with expression: %j, and ctxt: %j resulting: %j', + currentAttr.name, + currentAttr.expression, + jexlctxt, + valueExpression + ); } - logger.debug( - context, - 'Evaluated attr: %j, with expression: %j, and ctxt: %j resulting: %j', - currentAttr.name, - currentAttr.expression, - jexlctxt, - valueExpression - ); - } - currentAttr.hitted = hitted; - currentAttr.value = valueExpression; + currentAttr.hitted = hitted; + currentAttr.value = valueExpression; - //store de New Attributte in entity data structure - if (hitted === true) { - if (entities[attrEntityName] === undefined) { - entities[attrEntityName] = {}; - } - if (entities[attrEntityName][attrEntityType] === undefined) { - entities[attrEntityName][attrEntityType] = []; + //store de New Attributte in entity data structure + if (hitted === true) { + if (entities[attrEntityName] === undefined) { + entities[attrEntityName] = {}; + } + if (entities[attrEntityName][attrEntityType] === undefined) { + entities[attrEntityName][attrEntityType] = []; + } + //store de New Attributte + entities[attrEntityName][attrEntityType].push(currentAttr); } - //store de New Attributte - entities[attrEntityName][attrEntityType].push(currentAttr); - } - //RE-Populate de JEXLcontext (except for null or NaN we preffer undefined) - jexlctxt[currentAttr.name] = valueExpression; + //RE-Populate de JEXLcontext (except for null or NaN we preffer undefined) + jexlctxt[currentAttr.name] = valueExpression; - // Expand metadata value expression - if (currentAttr.metadata) { - for (var metaKey in currentAttr.metadata) { - if (currentAttr.metadata[metaKey].expression && metaKey !== constants.TIMESTAMP_ATTRIBUTE) { - let newAttrMeta = {}; - if (currentAttr.metadata[metaKey].type) { - newAttrMeta['type'] = currentAttr.metadata[metaKey].type; - } - let metaValueExpression; - try { - metaValueExpression = jexlParser.applyExpression( - currentAttr.metadata[metaKey].expression, - jexlctxt, - typeInformation - ); - //we fallback to null if anything unexpecte happend - if ( - metaValueExpression === null || - metaValueExpression === undefined || - Number.isNaN(metaValueExpression) - ) { + // Expand metadata value expression + if (currentAttr.metadata) { + for (var metaKey in currentAttr.metadata) { + if (currentAttr.metadata[metaKey].expression && metaKey !== constants.TIMESTAMP_ATTRIBUTE) { + let newAttrMeta = {}; + if (currentAttr.metadata[metaKey].type) { + newAttrMeta['type'] = currentAttr.metadata[metaKey].type; + } + let metaValueExpression; + try { + metaValueExpression = jexlParser.applyExpression( + currentAttr.metadata[metaKey].expression, + jexlctxt, + typeInformation + ); + //we fallback to null if anything unexpecte happend + if ( + metaValueExpression === null || + metaValueExpression === undefined || + Number.isNaN(metaValueExpression) + ) { + metaValueExpression = null; + } + } catch (e) { metaValueExpression = null; } - } catch (e) { - metaValueExpression = null; + newAttrMeta['value'] = metaValueExpression; + currentAttr.metadata[metaKey] = newAttrMeta; } - newAttrMeta['value'] = metaValueExpression; - currentAttr.metadata[metaKey] = newAttrMeta; } } } - } - //now we can compute explicit (Bool or Array) with the complete JexlContext - let explicit = false; - if (typeof typeInformation.explicitAttrs === 'string') { - try { - explicit = jexlParser.applyExpression(typeInformation.explicitAttrs, jexlctxt, typeInformation); - if (explicit instanceof Array && mustInsertTimeInstant) { - explicit.push(constants.TIMESTAMP_ATTRIBUTE); + //now we can compute explicit (Bool or Array) with the complete JexlContext + let explicit = false; + if (typeof typeInformation.explicitAttrs === 'string') { + try { + explicit = jexlParser.applyExpression(typeInformation.explicitAttrs, jexlctxt, typeInformation); + if (explicit instanceof Array && mustInsertTimeInstant) { + explicit.push(constants.TIMESTAMP_ATTRIBUTE); + } + logger.debug( + context, + 'Calculated explicitAttrs with expression: %j and ctxt: %j resulting: %j', + typeInformation.explicitAttrs, + jexlctxt, + explicit + ); + } catch (e) { + // nothing to do: exception is already logged at info level } - logger.debug( - context, - 'Calculated explicitAttrs with expression: %j and ctxt: %j resulting: %j', - typeInformation.explicitAttrs, - jexlctxt, - explicit - ); - } catch (e) { - // nothing to do: exception is already logged at info level + } else if (typeof typeInformation.explicitAttrs == 'boolean') { + explicit = typeInformation.explicitAttrs; } - } else if (typeof typeInformation.explicitAttrs == 'boolean') { - explicit = typeInformation.explicitAttrs; - } - - //more mesures may be added to the attribute list (unnhandled/left mesaures) l - if (explicit === false && Object.keys(measures).length > 0) { - entities[entityName][typeInformation.type] = entities[entityName][typeInformation.type].concat(measures); - } - - //PRE-PROCESSING FINISHED - //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload - //Get ready to build and send NGSI payload (entities-->payload) - payload.actionType = 'append'; + //more mesures may be added to the attribute list (unnhandled/left mesaures) l + if (explicit === false && Object.keys(measures).length > 0) { + entities[entityName][typeInformation.type] = entities[entityName][typeInformation.type].concat(measures); + } - payload.entities = []; - const currentIsoDate = new Date().toISOString(); - const currentMoment = moment(currentIsoDate); - for (let ename in entities) { - for (let etype in entities[ename]) { - let e = {}; - e.id = String(ename); - e.type = String(etype); - let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. - let timestampAttrs = null; - if (mustInsertTimeInstant) { - // get timestamp for current entity - - timestampAttrs = entities[ename][etype].filter((item) => item.name === constants.TIMESTAMP_ATTRIBUTE); - if (timestampAttrs && timestampAttrs.length > 0) { - timestamp.value = timestampAttrs[0]['value']; - } + //PRE-PROCESSING FINISHED + //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload + + //Get ready to build and send NGSI payload (entities-->payload) + //payload.actionType = 'append'; + + //payload.entities = []; + //const currentIsoDate = new Date().toISOString(); + //const currentMoment = moment(currentIsoDate); + for (let ename in entities) { + for (let etype in entities[ename]) { + let e = {}; + e.id = String(ename); + e.type = String(etype); + let timestamp = { type: constants.TIMESTAMP_TYPE_NGSI2 }; //timestamp scafold-attr for insertions. + let timestampAttrs = null; + if (mustInsertTimeInstant) { + // get timestamp for current entity - if (timestamp.value) { - if (!moment(timestamp.value, moment.ISO_8601, true).isValid()) { - callback(new errors.BadTimestamp(timestamp.value, entityName, typeInformation)); - return; + timestampAttrs = entities[ename][etype].filter( + (item) => item.name === constants.TIMESTAMP_ATTRIBUTE + ); + if (timestampAttrs && timestampAttrs.length > 0) { + timestamp.value = timestampAttrs[0]['value']; } - } else { - if (!typeInformation.timezone) { - timestamp.value = currentIsoDate; - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + + if (timestamp.value) { + if (!moment(timestamp.value, moment.ISO_8601, true).isValid()) { + callback(new errors.BadTimestamp(timestamp.value, entityName, typeInformation)); + return; + } } else { - timestamp.value = currentMoment - .tz(typeInformation.timezone) - .format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); - jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + if (!typeInformation.timezone) { + timestamp.value = currentIsoDate; + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + } else { + timestamp.value = currentMoment + .tz(typeInformation.timezone) + .format('YYYY-MM-DD[T]HH:mm:ss.SSSZ'); + jexlctxt[constants.TIMESTAMP_ATTRIBUTE] = timestamp.value; + } } } - } - //extract attributes - let isEmpty = true; - for (let attr of entities[ename][etype]) { - if ( - attr.name !== 'id' && - attr.name !== 'type' && - (attr.value !== attr.skipValue || attr.skipValue === undefined) && - (attr.hitted || attr.hitted === undefined) && //undefined is for pure measures - (typeof explicit === 'boolean' || //true and false already handled - (explicit instanceof Array && //check the array version - (explicit.includes(attr.name) || - explicit.some( - (item) => attr.object_id !== undefined && item.object_id === attr.object_id - )))) - ) { - isEmpty = false; - if (mustInsertTimeInstant) { - // Add TimeInstant to all attribute metadata of all entities - if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { - if (!attr.metadata) { - attr.metadata = {}; + //extract attributes + let isEmpty = true; + for (let attr of entities[ename][etype]) { + if ( + attr.name !== 'id' && + attr.name !== 'type' && + (attr.value !== attr.skipValue || attr.skipValue === undefined) && + (attr.hitted || attr.hitted === undefined) && //undefined is for pure measures + (typeof explicit === 'boolean' || //true and false already handled + (explicit instanceof Array && //check the array version + (explicit.includes(attr.name) || + explicit.some( + (item) => attr.object_id !== undefined && item.object_id === attr.object_id + )))) + ) { + isEmpty = false; + if (mustInsertTimeInstant) { + // Add TimeInstant to all attribute metadata of all entities + if (attr.name !== constants.TIMESTAMP_ATTRIBUTE) { + if (!attr.metadata) { + attr.metadata = {}; + } + attr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; } - attr.metadata[constants.TIMESTAMP_ATTRIBUTE] = timestamp; } + e[attr.name] = { type: attr.type, value: attr.value, metadata: attr.metadata }; } - e[attr.name] = { type: attr.type, value: attr.value, metadata: attr.metadata }; } - } - if (!isEmpty) { - if (mustInsertTimeInstant) { - e[constants.TIMESTAMP_ATTRIBUTE] = timestamp; + if (!isEmpty) { + if (mustInsertTimeInstant) { + e[constants.TIMESTAMP_ATTRIBUTE] = timestamp; + } + payload.entities.push(e); } - payload.entities.push(e); } } - } + } // end for (let measures of originMeasures) let url = '/v2/op/update'; let options = NGSIUtils.createRequestObject(url, typeInformation, token); @@ -620,10 +639,14 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call // Note that the options object is prepared for the second case (multi entity), so we "patch" it // only in the first case - //Multientity more than one name o more than one type at primary entity - let multientity = Object.keys(entities).length > 1 || Object.keys(entities[entityName]).length > 1; + //Multi: multientity (more than one name o more than one type at primary entity) + // of multimeasure (originMeasures is an array of more than one element) + let multi = + Object.keys(entities).length > 1 || + Object.keys(entities[entityName]).length > 1 || + originMeasures.length > 1; - if (!multientity) { + if (!multi) { // recreate options object to use single entity update url = '/v2/entities?options=upsert'; options = NGSIUtils.createRequestObject(url, typeInformation, token); From ad5f514fd72e7bca15d0cba008da1a32031b181b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 6 May 2024 13:07:51 +0200 Subject: [PATCH 34/83] add test --- test/functional/testCases.js | 66 ++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 9f88f75d1..3d9a887b9 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -513,6 +513,72 @@ const testCases = [ } ] }, + { + describeName: '0022 Simple group with active attributes and multimeasures', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + commands: [], + lazy: [], + attributes: [ + { + object_id: 'a', + name: 'attr_a', + type: 'Number' + } + ], + static_attributes: [] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + loglevel: 'debug', + shouldName: + 'A - WHEN sending defined object_ids (measures) through http IT should send measures to Context Broker preserving value types and name mappings', + type: 'multimeasure', // TBD: this should be implemented to expect /v2/op/update + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: [ + [ + { + a: 0 + } + ], + [ + { + a: 6 + } + ] + ] + }, + expectation: { + id: globalEnv.entity_name, + type: globalEnv.entity_type, + attr_a: { + value: 6, + type: 'Number' + } + } + } + ] + }, // 0100 - JEXL TESTS { describeName: '0100 - Simple group with active attribute + JEXL expression boolean (!)', From 292c3389dacd4a70f5b550ec20ce1b0e1f025334 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 6 May 2024 15:14:23 +0200 Subject: [PATCH 35/83] allow change default express limit --- config.js | 3 ++- lib/commonConfig.js | 8 +++++++- lib/services/northBound/northboundServer.js | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/config.js b/config.js index 3542fc7e2..88b4e610d 100644 --- a/config.js +++ b/config.js @@ -76,7 +76,8 @@ var config = { subservice: '/gardens', providerUrl: 'http://192.168.56.1:4041', deviceRegistrationDuration: 'P1M', - defaultType: 'Thing' + defaultType: 'Thing', + expressLimit: '1Mb' }; module.exports = config; diff --git a/lib/commonConfig.js b/lib/commonConfig.js index d7f4c2c40..d16908615 100644 --- a/lib/commonConfig.js +++ b/lib/commonConfig.js @@ -156,7 +156,8 @@ function processEnvironmentVariables() { 'IOTA_FALLBACK_TENANT', 'IOTA_FALLBACK_PATH', 'IOTA_LD_SUPPORT_NULL', - 'IOTA_LD_SUPPORT_DATASET_ID' + 'IOTA_LD_SUPPORT_DATASET_ID', + 'IOTA_EXPRESS_LIMIT' ]; const iotamVariables = [ 'IOTA_IOTAM_URL', @@ -468,6 +469,11 @@ function processEnvironmentVariables() { ? config.defaultEntityNameConjunction : ':'; } + if (process.env.IOTA_EXPRESS_LIMIT) { + config.expressLimit = process.env.IOTA_EXPRESS_LIMIT; + } else { + config.expressLimit = config.expressLimit ? config.expressLimit : '1mb'; + } } function setConfig(newConfig) { diff --git a/lib/services/northBound/northboundServer.js b/lib/services/northBound/northboundServer.js index 5ed44f8c4..bd22879d2 100644 --- a/lib/services/northBound/northboundServer.js +++ b/lib/services/northBound/northboundServer.js @@ -56,8 +56,8 @@ function start(config, callback) { northboundServer.app.set('port', config.server.port); northboundServer.app.set('host', config.server.host || '0.0.0.0'); northboundServer.app.use(domainUtils.requestDomain); - northboundServer.app.use(bodyParser.json()); - northboundServer.app.use(bodyParser.json({ type: 'application/*+json' })); + northboundServer.app.use(bodyParser.json({ limit: config.expressLimit })); + northboundServer.app.use(bodyParser.json({ type: 'application/*+json', limit: config.expressLimit })); if (config.logLevel && config.logLevel === 'DEBUG') { northboundServer.app.use(middlewares.traceRequest); From 9dca90510c52159fcfe57702cd4aafa22c34ee31 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 6 May 2024 15:23:55 +0200 Subject: [PATCH 36/83] update CNR --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e69de29bb..bf9eeeba0 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -0,0 +1 @@ +- Fix default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (#telefonicaid/iotagent-json#827) From 62446dff48a199f4ff6c26941dfdc5085b9e72e3 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 6 May 2024 15:28:47 +0200 Subject: [PATCH 37/83] update doc --- doc/admin.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/doc/admin.md b/doc/admin.md index 413da4384..79939292d 100644 --- a/doc/admin.md +++ b/doc/admin.md @@ -260,13 +260,13 @@ the `mongob` section (as described bellow). E.g.: It configures the MongoDB driver for those repositories with 'mongodb' type. If the `host` parameter is a list of comma-separated IPs, they will be considered to be part of a Replica Set. In that case, the optional property -`replicaSet` should contain the Replica Set name. If the database requires authentication, username (`user`), -password (`password`) and authSource (`authSource`) can be set. If the database requires TLS/SSL connection but any -validation of the certificate chain is not mandatory, all you need is to set the ssl (`ssl`) option as `true` to connect -the database. If you need to add more complex option(s) such as `retryWrites=true` or `w=majority` when connection -database, extraArgs (`extraArgs`) can be used to perform it. For The MongoBD driver will retry the connection at startup -time `retries` times, waiting `retryTime` seconds between attempts, if those attributes are present (default values are -5 and 5 respectively). E.g.: +`replicaSet` should contain the Replica Set name. If the database requires authentication, username (`user`), password +(`password`) and authSource (`authSource`) can be set. If the database requires TLS/SSL connection but any validation of +the certificate chain is not mandatory, all you need is to set the ssl (`ssl`) option as `true` to connect the database. +If you need to add more complex option(s) such as `retryWrites=true` or `w=majority` when connection database, extraArgs +(`extraArgs`) can be used to perform it. For The MongoBD driver will retry the connection at startup time `retries` +times, waiting `retryTime` seconds between attempts, if those attributes are present (default values are 5 and 5 +respectively). E.g.: ```javascript { @@ -421,6 +421,12 @@ characters (such as semi-colons) which are specification. When provisioning devices, it is necessary that the developer provides valid `objectId`-`name` mappings whenever relaxed mode is used, to prevent the consumption of forbidden characters. +#### `expressLimit` + +IotAgents, as all Express applications that use the body-parser middleware, have a default limit to the request body +size that the application will handle. This default limit for ioiotagnets are 1Mb. So, if your IotAgent receives a +request with a body that exceeds this limit, the application will throw a “Error: Request entity too large”. + ### Configuration using environment variables Some of the configuration parameters can be overriden with environment variables, to ease the use of those parameters @@ -482,6 +488,7 @@ overrides. | IOTA_EXPLICIT_ATTRS | `explicitAttrs` | | IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION | `defaultEntityNameConjunction` | | IOTA_RELAX_TEMPLATE_VALIDATION | `relaxTemplateValidation` | +| IOTA_EXPRESS_LIMIT | `expressLimit` | Note: From ad1460bff65c47aa1217676edd6f074e365e6f57 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 6 May 2024 16:00:20 +0200 Subject: [PATCH 38/83] update CNR --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index e69de29bb..0d4ce7bcd 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -0,0 +1 @@ +- Fix: allow send multiple measures to CB in a batch (v2/op/update) instead of using multiples single request (iotagent-json#825) From fd1a7d4d1d3c421e19c0a0aec82d3a8b8bae921b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 6 May 2024 16:56:11 +0200 Subject: [PATCH 39/83] Apply suggestions from code review --- CHANGES_NEXT_RELEASE | 2 +- doc/admin.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index bf9eeeba0..25e0be54d 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1 @@ -- Fix default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (#telefonicaid/iotagent-json#827) +- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (telefonicaid/iotagent-json#827) diff --git a/doc/admin.md b/doc/admin.md index 79939292d..644e3ace7 100644 --- a/doc/admin.md +++ b/doc/admin.md @@ -427,6 +427,8 @@ IotAgents, as all Express applications that use the body-parser middleware, have size that the application will handle. This default limit for ioiotagnets are 1Mb. So, if your IotAgent receives a request with a body that exceeds this limit, the application will throw a “Error: Request entity too large”. +The 1Mb default can be changed setting the `expressLimit` configuration parameter (or equivalente `IOTA_EXPRESS_LIMIT` environment variable). + ### Configuration using environment variables Some of the configuration parameters can be overriden with environment variables, to ease the use of those parameters From f3cb6e9163050a1df46625c91b5ac13a18b8ea2b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 20:43:35 +0000 Subject: [PATCH 40/83] Bump jinja2 from 3.1.3 to 3.1.4 in /doc Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.3 to 3.1.4. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/jinja/compare/3.1.3...3.1.4) --- updated-dependencies: - dependency-name: jinja2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- doc/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 4252df910..b362ac268 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,4 @@ mkdocs==1.2.4 Pygments==2.15.0 Markdown==3.3.4 -jinja2==3.1.3 +jinja2==3.1.4 From 751c4e51a295d30046d093ce277a01b1287c2aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 7 May 2024 13:09:38 +0200 Subject: [PATCH 41/83] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 059f36ea1..018d07f7a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Fix: allow send multiple measures to CB in a batch (v2/op/update) instead of using multiples single request (iotagent-json#825) +- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) instead of using multiples single request (iotagent-json#825) - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (telefonicaid/iotagent-json#827) \ No newline at end of file From ecf1e04743ef29258e40fef0603d24175fe02c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 7 May 2024 13:10:10 +0200 Subject: [PATCH 42/83] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 018d07f7a..5d84eee8d 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ - Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) instead of using multiples single request (iotagent-json#825) -- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (telefonicaid/iotagent-json#827) \ No newline at end of file +- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) \ No newline at end of file From ae17e28b0c5574c3c103e9ee01e7b045fc8ba4a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Tue, 7 May 2024 13:10:54 +0200 Subject: [PATCH 43/83] Update lib/services/ngsi/entities-NGSI-v2.js --- lib/services/ngsi/entities-NGSI-v2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index d8514b6ac..5497f5a0e 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -299,7 +299,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token originMeasures = [originMeasures]; } for (let measures of originMeasures) { - entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entoties data striucture + entities = {}; //{entityName:{entityType:[attrs]}} //SubGoal Populate entities data structure let jexlctxt = {}; //will store the whole context (not just for JEXL) let plainMeasures = null; //will contain measures POJO From 8c7ebcdacee9dcbb6b83847201b9d04899120fd0 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 7 May 2024 14:22:56 +0200 Subject: [PATCH 44/83] Update entities-NGSI-v2.js --- lib/services/ngsi/entities-NGSI-v2.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 5497f5a0e..3cb3b666a 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -545,12 +545,6 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token //PRE-PROCESSING FINISHED //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload - //Get ready to build and send NGSI payload (entities-->payload) - //payload.actionType = 'append'; - - //payload.entities = []; - //const currentIsoDate = new Date().toISOString(); - //const currentMoment = moment(currentIsoDate); for (let ename in entities) { for (let etype in entities[ename]) { let e = {}; From cfbe12fb428a494b37c4f1f5ceaeeac227cdafd0 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 7 May 2024 15:28:51 +0200 Subject: [PATCH 45/83] Update entities-NGSI-v2.js --- lib/services/ngsi/entities-NGSI-v2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 3cb3b666a..8adc42b16 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -544,7 +544,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token //PRE-PROCESSING FINISHED //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload - + //Get ready to build and send NGSI payload (entities-->payload) for (let ename in entities) { for (let etype in entities[ename]) { let e = {}; From 3705bd68e31907d4a1c1822c5d3263c1b2772e12 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 8 May 2024 11:36:28 +0200 Subject: [PATCH 46/83] fix and clone typeInformation --- lib/services/ngsi/entities-NGSI-v2.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 8adc42b16..e91e6e37e 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -260,7 +260,7 @@ function sendQueryValueNgsi2(entityName, attributes, typeInformation, token, cal * @param {Object} typeInformation Configuration information for the device. * @param {String} token User token to identify against the PEP Proxies (optional). */ -function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token, callback) { +function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, token, callback) { //aux function used to builf JEXL context. //it returns a flat object from an Attr array function reduceAttrToPlainObject(attrs, initObj = {}) { @@ -273,11 +273,10 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token return initObj; } } - + //Make a clone and overwrite + let typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); - //Make a clone and overwrite - typeInformation = JSON.parse(JSON.stringify(typeInformation)); //Check mandatory information: type if (!typeInformation || !typeInformation.type) { callback(new errors.TypeNotFound(null, entityName, typeInformation)); @@ -303,6 +302,8 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token let jexlctxt = {}; //will store the whole context (not just for JEXL) let plainMeasures = null; //will contain measures POJO + //Make a clone and overwrite + typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); //Rename all measures with matches with id and type to measure_id and measure_type for (let measure of measures) { @@ -545,6 +546,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, typeInformation, token //PRE-PROCESSING FINISHED //Explicit ATTRS and SKIPVALUES will be managed while we build NGSI payload //Get ready to build and send NGSI payload (entities-->payload) + for (let ename in entities) { for (let etype in entities[ename]) { let e = {}; From 94db695ed0dc7248cb4e4e1572f8069a07dad88c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 8 May 2024 11:45:20 +0200 Subject: [PATCH 47/83] Update multimeasure functional teste --- test/functional/testCases.js | 43 ++++++++++++--------- test/functional/testUtils.js | 73 ++++++++++++++++++++++++++---------- 2 files changed, 79 insertions(+), 37 deletions(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 3d9a887b9..a8dcd2f71 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -544,7 +544,7 @@ const testCases = [ }, should: [ { - loglevel: 'debug', + //loglevel: 'debug', shouldName: 'A - WHEN sending defined object_ids (measures) through http IT should send measures to Context Broker preserving value types and name mappings', type: 'multimeasure', // TBD: this should be implemented to expect /v2/op/update @@ -556,25 +556,34 @@ const testCases = [ k: globalEnv.apikey }, json: [ - [ - { - a: 0 - } - ], - [ - { - a: 6 - } - ] + { + a: 0 + }, + { + a: 6 + } ] }, expectation: { - id: globalEnv.entity_name, - type: globalEnv.entity_type, - attr_a: { - value: 6, - type: 'Number' - } + actionType: 'append', + entities: [ + { + attr_a: { + type: 'Number', + value: 0 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 6 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + } + ] } } ] diff --git a/test/functional/testUtils.js b/test/functional/testUtils.js index f41e8433a..495e8d470 100644 --- a/test/functional/testUtils.js +++ b/test/functional/testUtils.js @@ -98,28 +98,61 @@ function sendMeasureIotaLib(measure, provision) { * @param {Object} json * @returns {Array} measures */ -function jsonToIotaMeasures(json) { - let measures = []; - for (let key in json) { - /* eslint-disable-next-line no-prototype-builtins */ - if (json.hasOwnProperty(key)) { - let measure = { - name: key, - value: json[key] - }; - // A bit of Magic. If the key is TimeInstant, we set the type to DateTime. - // When sending the data through iot - if (key === 'TimeInstant') { - measure.type = 'DateTime'; - } else { - // Although the type is not meaningfull and we could have picked any string for this, - // we have aligned with DEFAULT_ATTRIBUTE_TYPE constant in IOTA-JSON and IOTA-UL repositories - measure.type = 'Text'; +function jsonToIotaMeasures(originJson) { + // FIXME: maybe this could be refactored to use less code + if (originJson && originJson[0]) { + // multimeasure case + let finalMeasures = []; + + for (let json of originJson) { + let measures = []; + for (let key in json) { + /* eslint-disable-next-line no-prototype-builtins */ + if (json.hasOwnProperty(key)) { + let measure = { + name: key, + value: json[key] + }; + // A bit of Magic. If the key is TimeInstant, we set the type to DateTime. + // When sending the data through iot + if (key === 'TimeInstant') { + measure.type = 'DateTime'; + } else { + // Although the type is not meaningfull and we could have picked any string for this, + // we have aligned with DEFAULT_ATTRIBUTE_TYPE constant in IOTA-JSON and IOTA-UL repositories + measure.type = 'Text'; + } + measures.push(measure); + } + } + finalMeasures.push(measures); + } + return finalMeasures; + } else { + let json = originJson; + + let measures = []; + for (let key in json) { + /* eslint-disable-next-line no-prototype-builtins */ + if (json.hasOwnProperty(key)) { + let measure = { + name: key, + value: json[key] + }; + // A bit of Magic. If the key is TimeInstant, we set the type to DateTime. + // When sending the data through iot + if (key === 'TimeInstant') { + measure.type = 'DateTime'; + } else { + // Although the type is not meaningfull and we could have picked any string for this, + // we have aligned with DEFAULT_ATTRIBUTE_TYPE constant in IOTA-JSON and IOTA-UL repositories + measure.type = 'Text'; + } + measures.push(measure); } - measures.push(measure); } + return measures; } - return measures; } /** @@ -170,7 +203,7 @@ async function testCase(measure, expectation, provision, env, config, type, tran let receivedContext = []; let cbMockRoute = ''; // Set the correct route depending if the test is multientity or not - if (type === 'multientity') { + if (type === 'multientity' || type === 'multimeasure') { cbMockRoute = '/v2/op/update'; } else { cbMockRoute = '/v2/entities?options=upsert'; From 328ee2ed65e671e55dac560021676134f0347603 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 8 May 2024 12:00:04 +0200 Subject: [PATCH 48/83] update test --- test/functional/testCases.js | 55 ++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index a8dcd2f71..a0a040b2a 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -559,6 +559,21 @@ const testCases = [ { a: 0 }, + { + a: 1 + }, + { + a: 2 + }, + { + a: 3 + }, + { + a: 4 + }, + { + a: 5 + }, { a: 6 } @@ -575,6 +590,46 @@ const testCases = [ id: globalEnv.entity_name, type: globalEnv.entity_type }, + { + attr_a: { + type: 'Number', + value: 1 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 2 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 3 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 4 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 5 + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, { attr_a: { type: 'Number', From 9034c0254970e75bd0ec39d673a5d0c4f510c33b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 13 May 2024 13:16:54 +0200 Subject: [PATCH 49/83] clean old comment --- test/functional/testCases.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index a0a040b2a..6cf4d064e 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -544,10 +544,9 @@ const testCases = [ }, should: [ { - //loglevel: 'debug', shouldName: 'A - WHEN sending defined object_ids (measures) through http IT should send measures to Context Broker preserving value types and name mappings', - type: 'multimeasure', // TBD: this should be implemented to expect /v2/op/update + type: 'multimeasure', measure: { url: 'http://localhost:' + config.http.port + '/iot/json', method: 'POST', From 1bbd16897e22000fe7bed05af461452b2a7c2fbf Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 14 May 2024 11:23:37 +0200 Subject: [PATCH 50/83] avoid duplicate origintypeInformation --- lib/services/ngsi/entities-NGSI-v2.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index e91e6e37e..1870a997f 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -274,12 +274,11 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, } } //Make a clone and overwrite - let typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); - let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(typeInformation); + let idTypeSSSList = pluginUtils.getIdTypeServSubServiceFromDevice(originTypeInformation); //Check mandatory information: type - if (!typeInformation || !typeInformation.type) { - callback(new errors.TypeNotFound(null, entityName, typeInformation)); + if (!originTypeInformation || !originTypeInformation.type) { + callback(new errors.TypeNotFound(null, entityName, originTypeInformation)); return; } @@ -291,7 +290,8 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, const currentIsoDate = new Date().toISOString(); const currentMoment = moment(currentIsoDate); //Managing timestamp (mustInsertTimeInstant flag to decide if we should insert Timestamp later on) - const mustInsertTimeInstant = typeInformation.timestamp !== undefined ? typeInformation.timestamp : false; + const mustInsertTimeInstant = + originTypeInformation.timestamp !== undefined ? originTypeInformation.timestamp : false; // Check if measures is a single measure or a array of measures (a multimeasure) if (originMeasures[0] && !originMeasures[0][0]) { @@ -620,7 +620,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, } // end for (let measures of originMeasures) let url = '/v2/op/update'; - let options = NGSIUtils.createRequestObject(url, typeInformation, token); + let options = NGSIUtils.createRequestObject(url, originTypeInformation, token); options.json = payload; // Prevent to update an entity with an empty payload: more than id and type @@ -645,7 +645,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, if (!multi) { // recreate options object to use single entity update url = '/v2/entities?options=upsert'; - options = NGSIUtils.createRequestObject(url, typeInformation, token); + options = NGSIUtils.createRequestObject(url, originTypeInformation, token); delete payload.actionType; let entityAttrs = payload.entities[0]; @@ -670,7 +670,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, request( options, - generateNGSI2OperationHandler('update', entityName, typeInformation, token, options, callback) + generateNGSI2OperationHandler('update', entityName, originTypeInformation, token, options, callback) ); } else { logger.debug( From 8d7fea1e2a003b2b7d648b7b1a794722edc3fb8f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 14 May 2024 11:34:52 +0200 Subject: [PATCH 51/83] declare typeInformation into loop --- lib/services/ngsi/entities-NGSI-v2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 1870a997f..5330ca9d8 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -303,7 +303,7 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, let plainMeasures = null; //will contain measures POJO //Make a clone and overwrite - typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); + let typeInformation = JSON.parse(JSON.stringify(originTypeInformation)); //Rename all measures with matches with id and type to measure_id and measure_type for (let measure of measures) { From c7426811b164631d200d48372527b783ced77e78 Mon Sep 17 00:00:00 2001 From: mapedraza <40356341+mapedraza@users.noreply.github.com> Date: Tue, 14 May 2024 15:47:33 +0200 Subject: [PATCH 52/83] Remove 533 tests --- test/functional/testCases.js | 101 ----------------------------------- 1 file changed, 101 deletions(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 2232d2a10..76993086d 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -3126,107 +3126,6 @@ const testCases = [ } ] }, - { - describeName: - '0533 - Group with explicit attrs:[ ] (empty array) + active attributes + TimeInstant attribute + static attributes + timestamp:true', - provision: { - url: 'http://localhost:' + config.iota.server.port + '/iot/services', - method: 'POST', - json: { - services: [ - { - resource: '/iot/json', - apikey: globalEnv.apikey, - entity_type: globalEnv.entity_type, - explicitAttrs: "['DateIssued']", - timestamp: true, - commands: [], - lazy: [], - attributes: [ - { - name: 'attr_a', - object_id: 'a', - type: 'Number' - }, - { - name: 'attr_b', - object_id: 'b', - type: 'Number' - }, - { - name: 'DateIssued', - object_id: 'TimeInstant', - type: 'DateTime' - }, - { - name: 'TimeInstant', - object_id: 't', - type: 'DateTime' - } - ], - static_attributes: [ - { - name: 'static_a', - type: 'Number', - value: 3 - }, - { - name: 'static_b', - type: 'Number', - value: 4 - } - ] - } - ] - }, - headers: { - 'fiware-service': globalEnv.service, - 'fiware-servicepath': globalEnv.servicePath - } - }, - should: [ - { - shouldName: - 'A - WHEN sending data and a measure called "t" (defined as TimeInstant attribte) through http IT should NOT store anything into the Context Broker (No request to CB)', - type: 'single', - measure: { - url: 'http://localhost:' + config.http.port + '/iot/json', - method: 'POST', - qs: { - i: globalEnv.deviceId, - k: globalEnv.apikey - }, - json: { - a: 3, - b: 10, - c: 11, - t: '2015-12-14T08:06:01.468Z' - } - }, - expectation: {} - }, - { - shouldName: - 'B - WHEN sending data and a measure called TimeInstant through http IT should NOT store anything into the Context Broker (No request to CB)', - type: 'single', - measure: { - url: 'http://localhost:' + config.http.port + '/iot/json', - method: 'POST', - qs: { - i: globalEnv.deviceId, - k: globalEnv.apikey - }, - json: { - a: 3, - b: 10, - c: 11, - TimeInstant: '2015-12-14T08:06:01.468Z' - } - }, - expectation: {} // No payload expected - } - ] - }, { describeName: '0540 - Group with explicit attrs: JEXL expression based on measure resulting boolean + active attributes + static attributes', From 4e583694ead625339e3e2cb7bd9177def78abe60 Mon Sep 17 00:00:00 2001 From: mapedraza <40356341+mapedraza@users.noreply.github.com> Date: Tue, 14 May 2024 15:50:24 +0200 Subject: [PATCH 53/83] add 533 --- test/functional/testCases.js | 101 +++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 9f88f75d1..da12394c6 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2968,6 +2968,107 @@ const testCases = [ } ] }, + { + describeName: + '0533 - Group with explicit attrs:[ ] (empty array) + active attributes + TimeInstant attribute + static attributes + timestamp:true', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + explicitAttrs: "['DateIssued']", + timestamp: true, + commands: [], + lazy: [], + attributes: [ + { + name: 'attr_a', + object_id: 'a', + type: 'Number' + }, + { + name: 'attr_b', + object_id: 'b', + type: 'Number' + }, + { + name: 'DateIssued', + object_id: 'TimeInstant', + type: 'DateTime' + }, + { + name: 'TimeInstant', + object_id: 't', + type: 'DateTime' + } + ], + static_attributes: [ + { + name: 'static_a', + type: 'Number', + value: 3 + }, + { + name: 'static_b', + type: 'Number', + value: 4 + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending data and a measure called "t" (defined as TimeInstant attribte) through http IT should NOT store anything into the Context Broker (No request to CB)', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 3, + b: 10, + c: 11, + t: '2015-12-14T08:06:01.468Z' + } + }, + expectation: {} + }, + { + shouldName: + 'B - WHEN sending data and a measure called TimeInstant through http IT should NOT store anything into the Context Broker (No request to CB)', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 3, + b: 10, + c: 11, + TimeInstant: '2015-12-14T08:06:01.468Z' + } + }, + expectation: {} // No payload expected + } + ] + }, { describeName: '0540 - Group with explicit attrs: JEXL expression based on measure resulting boolean + active attributes + static attributes', From 53bc93a8b22507318122f1ce939ef1a67fcc7379 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 16 May 2024 09:35:47 +0200 Subject: [PATCH 54/83] add tests about sorted multimeasures by TimeInstant --- test/functional/testCases.js | 132 +++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index 6cf4d064e..cc7169c35 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -639,6 +639,138 @@ const testCases = [ } ] } + }, + { + shouldName: + 'A - WHEN sending defined object_ids (measures) through http IT should send measures with TimeInstant to Context Broker preserving value types and name mappings', + type: 'multimeasure', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: [ + { + a: 0, + TimeInstant: '2024-04-10T10:00:00Z' + }, + { + a: 1, + TimeInstant: '2024-04-10T10:05:00Z' + }, + { + a: 2, + TimeInstant: '2024-04-10T10:10:00Z' + }, + { + a: 3, + TimeInstant: '2024-04-10T10:15:00Z' + }, + { + a: 4, + TimeInstant: '2024-04-10T10:20:00Z' + }, + { + a: 5, + TimeInstant: '2024-04-10T10:25:00Z' + }, + { + a: 6, + TimeInstant: '2024-04-10T10:30:00Z' + } + ] + }, + expectation: { + actionType: 'append', + entities: [ + { + attr_a: { + type: 'Number', + value: 0 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:00:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 1 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:05:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 2 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:10:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 3 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:15:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 4 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:20:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 5 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:25:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 6 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:30:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + } + ] + } } ] }, From b6a7d234fb7dc585744376c9828840548fc47acf Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 16 May 2024 11:17:24 +0200 Subject: [PATCH 55/83] sort entities by TimeInstant --- lib/services/ngsi/entities-NGSI-v2.js | 17 ++- test/functional/testCases.js | 162 +++++++++++++++++++++++--- 2 files changed, 163 insertions(+), 16 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index 5330ca9d8..bb98bf864 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -662,7 +662,22 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, transformedObject.type = entityAttrs.type; options.json = transformedObject; options.method = 'POST'; - } // else: keep current options object created for a batch update + } else { + // keep current options object created for a batch update + // but try sort entities by TimeInstant + if (payload.entities.every((entity) => 'TimeInstant' in entity)) { + payload.entities.sort( + (a, b) => new Date(a.TimeInstant.value).getTime() - new Date(b.TimeInstant.value).getTime() + ); + options.json = payload; + } else { + logger.debug( + context, + "some entities lack the 'TimeInstant' key. Sorting is not feasible: %j ", + payload.entities + ); + } + } //Send the NGSI request logger.debug(context, 'Updating device value in the Context Broker at: %j', options.url); diff --git a/test/functional/testCases.js b/test/functional/testCases.js index cc7169c35..4efc88c6e 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -446,7 +446,7 @@ const testCases = [ ] }, { - describeName: '0021 Simple group with active attributes with metadata', + describeName: '0021 - Simple group with active attributes with metadata', provision: { url: 'http://localhost:' + config.iota.server.port + '/iot/services', method: 'POST', @@ -514,7 +514,7 @@ const testCases = [ ] }, { - describeName: '0022 Simple group with active attributes and multimeasures', + describeName: '0022 - Simple group with active attributes and multimeasures', provision: { url: 'http://localhost:' + config.iota.server.port + '/iot/services', method: 'POST', @@ -642,7 +642,7 @@ const testCases = [ }, { shouldName: - 'A - WHEN sending defined object_ids (measures) through http IT should send measures with TimeInstant to Context Broker preserving value types and name mappings', + 'A - WHEN sending defined object_ids (measures) through http IT should send measures with TimeInstant to Context Broker preserving value types and name mappings and order', type: 'multimeasure', measure: { url: 'http://localhost:' + config.http.port + '/iot/json', @@ -771,6 +771,138 @@ const testCases = [ } ] } + }, + { + shouldName: + 'A - WHEN sending defined object_ids (measures) through http IT should send measures with TimeInstant to Context Broker preserving value types and name mappings and sorted by TimeInstant', + type: 'multimeasure', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: [ + { + a: 0, + TimeInstant: '2024-04-10T10:15:00Z' + }, + { + a: 1, + TimeInstant: '2024-04-10T10:05:00Z' + }, + { + a: 2, + TimeInstant: '2024-04-10T10:20:00Z' + }, + { + a: 3, + TimeInstant: '2024-04-10T10:00:00Z' + }, + { + a: 4, + TimeInstant: '2024-04-10T10:10:00Z' + }, + { + a: 5, + TimeInstant: '2024-04-10T10:30:00Z' + }, + { + a: 6, + TimeInstant: '2024-04-10T10:25:00Z' + } + ] + }, + expectation: { + actionType: 'append', + entities: [ + { + attr_a: { + type: 'Number', + value: 3 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:00:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 1 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:05:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 4 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:10:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 0 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:15:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 2 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:20:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 6 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:25:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + }, + { + attr_a: { + type: 'Number', + value: 5 + }, + TimeInstant: { + type: 'DateTime', + value: '2024-04-10T10:30:00Z' + }, + id: globalEnv.entity_name, + type: globalEnv.entity_type + } + ] + } } ] }, @@ -2879,56 +3011,56 @@ const testCases = [ actionType: 'append', entities: [ { - id: globalEnv.entity_name, + id: 'TestType:TestDevice1', type: globalEnv.entity_type, - a: { + a1: { value: 23, type: 'Text', metadata: { TimeInstant: { - value: _.isDateString, + value: '2011-01-01T01:11:11.111Z', type: 'DateTime' } } }, TimeInstant: { - value: _.isDateString, + value: '2011-01-01T01:11:11.111Z', type: 'DateTime' } }, { - id: 'TestType:TestDevice1', + id: 'TestType:TestDevice2', type: globalEnv.entity_type, - a1: { + a2: { value: 23, type: 'Text', metadata: { TimeInstant: { - value: '2011-01-01T01:11:11.111Z', + value: '2022-02-02T02:22:22.222Z', type: 'DateTime' } } }, TimeInstant: { - value: '2011-01-01T01:11:11.111Z', + value: '2022-02-02T02:22:22.222Z', type: 'DateTime' } }, { - id: 'TestType:TestDevice2', + id: globalEnv.entity_name, type: globalEnv.entity_type, - a2: { + a: { value: 23, type: 'Text', metadata: { TimeInstant: { - value: '2022-02-02T02:22:22.222Z', + value: _.isDateString, type: 'DateTime' } } }, TimeInstant: { - value: '2022-02-02T02:22:22.222Z', + value: _.isDateString, type: 'DateTime' } } From 8ec17ceb4fe5e548404f257df8340c5173d5c64d Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 16 May 2024 11:27:06 +0200 Subject: [PATCH 56/83] fix linter --- lib/services/ngsi/entities-NGSI-v2.js | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index bb98bf864..56c0a9ede 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -662,21 +662,19 @@ function sendUpdateValueNgsi2(entityName, originMeasures, originTypeInformation, transformedObject.type = entityAttrs.type; options.json = transformedObject; options.method = 'POST'; + } else if (payload.entities.every((entity) => 'TimeInstant' in entity)) { + // Try sort entities by TimeInstant + payload.entities.sort( + (a, b) => new Date(a.TimeInstant.value).getTime() - new Date(b.TimeInstant.value).getTime() + ); + options.json = payload; } else { - // keep current options object created for a batch update - // but try sort entities by TimeInstant - if (payload.entities.every((entity) => 'TimeInstant' in entity)) { - payload.entities.sort( - (a, b) => new Date(a.TimeInstant.value).getTime() - new Date(b.TimeInstant.value).getTime() - ); - options.json = payload; - } else { - logger.debug( - context, - "some entities lack the 'TimeInstant' key. Sorting is not feasible: %j ", - payload.entities - ); - } + // keep current options object created for a batch update + logger.debug( + context, + "some entities lack the 'TimeInstant' key. Sorting is not feasible: %j ", + payload.entities + ); } //Send the NGSI request From b5bae06f3d97f420d72cd30984dc6ba66fbb9fca Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 16 May 2024 11:36:13 +0200 Subject: [PATCH 57/83] update CNR --- CHANGES_NEXT_RELEASE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 5d84eee8d..f5501faee 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) instead of using multiples single request (iotagent-json#825) -- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) \ No newline at end of file +- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825) +- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) From f364cc1658325d6a2258ceedfe92a37702944de3 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 17 May 2024 10:08:53 +0200 Subject: [PATCH 58/83] add test with empy array for explicitAttrs and timestamp false --- test/functional/testCases.js | 114 ++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 3 deletions(-) diff --git a/test/functional/testCases.js b/test/functional/testCases.js index da12394c6..ab5cfd4c5 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -2980,7 +2980,7 @@ const testCases = [ resource: '/iot/json', apikey: globalEnv.apikey, entity_type: globalEnv.entity_type, - explicitAttrs: "['DateIssued']", + explicitAttrs: '[ ]', timestamp: true, commands: [], lazy: [], @@ -3045,7 +3045,115 @@ const testCases = [ t: '2015-12-14T08:06:01.468Z' } }, - expectation: {} + expectation: { + id: 'TestType:TestDevice', + type: 'TestType', + TimeInstant: { + type: 'DateTime', + value: '2015-12-14T08:06:01.468Z' + } + } + }, + { + shouldName: + 'B - WHEN sending data and a measure called TimeInstant through http IT should NOT store anything into the Context Broker (No request to CB)', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 3, + b: 10, + c: 11, + TimeInstant: '2015-12-14T08:06:01.468Z' + } + }, + expectation: [] // No payload expected + } + ] + }, + { + describeName: + '0534 - Group with explicit attrs:[ ] (empty array) + active attributes + TimeInstant attribute + static attributes + timestamp:false', + provision: { + url: 'http://localhost:' + config.iota.server.port + '/iot/services', + method: 'POST', + json: { + services: [ + { + resource: '/iot/json', + apikey: globalEnv.apikey, + entity_type: globalEnv.entity_type, + explicitAttrs: '[ ]', + timestamp: false, + commands: [], + lazy: [], + attributes: [ + { + name: 'attr_a', + object_id: 'a', + type: 'Number' + }, + { + name: 'attr_b', + object_id: 'b', + type: 'Number' + }, + { + name: 'DateIssued', + object_id: 'TimeInstant', + type: 'DateTime' + }, + { + name: 'TimeInstant', + object_id: 't', + type: 'DateTime' + } + ], + static_attributes: [ + { + name: 'static_a', + type: 'Number', + value: 3 + }, + { + name: 'static_b', + type: 'Number', + value: 4 + } + ] + } + ] + }, + headers: { + 'fiware-service': globalEnv.service, + 'fiware-servicepath': globalEnv.servicePath + } + }, + should: [ + { + shouldName: + 'A - WHEN sending data and a measure called "t" (defined as TimeInstant attribte) through http IT should NOT store anything into the Context Broker (No request to CB)', + type: 'single', + measure: { + url: 'http://localhost:' + config.http.port + '/iot/json', + method: 'POST', + qs: { + i: globalEnv.deviceId, + k: globalEnv.apikey + }, + json: { + a: 3, + b: 10, + c: 11, + t: '2015-12-14T08:06:01.468Z' + } + }, + expectation: [] }, { shouldName: @@ -3065,7 +3173,7 @@ const testCases = [ TimeInstant: '2015-12-14T08:06:01.468Z' } }, - expectation: {} // No payload expected + expectation: [] // No payload expected } ] }, From d9d0c26fe2caed2e08136780e30204380273f2ad Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Fri, 17 May 2024 10:49:38 +0200 Subject: [PATCH 59/83] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index f5501faee..05feb0873 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825) +- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825, #1612) - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) From 9534cfaba0e8af203bb82e54fa7ec5e65c98487e Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 20 May 2024 10:33:24 +0200 Subject: [PATCH 60/83] fix md --- doc/api.md | 111 ++++++++++++++++++++++++++++------------------------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/doc/api.md b/doc/api.md index d661861c0..19970cfc7 100644 --- a/doc/api.md +++ b/doc/api.md @@ -11,7 +11,7 @@ - [Uniqueness of groups and devices](#uniqueness-of-groups-and-devices) - [Special measures and attributes names](#special-measures-and-attributes-names) - [Entity attributes](#entity-attributes) - - [Multientity support)](#multientity-support) + - [Multientity support](#multientity-support) - [Metadata support](#metadata-support) - [NGSI LD data and metadata considerations](#ngsi-ld-data-and-metadata-considerations) - [Advice on Attribute definitions](#advice-on-attribute-definitions) @@ -286,32 +286,37 @@ e.g.: ```json { - "entity_type": "Lamp", - "resource": "/iot/d", - "protocol": "PDI-IoTA-UltraLight", -..etc - "commands": [ - {"name": "on","type": "command"}, - {"name": "off","type": "command"} - ], - "attributes": [ - {"object_id": "s", "name": "state", "type":"Text"}, - {"object_id": "l", "name": "luminosity", "type":"Integer", - "metadata":{ - "unitCode":{"type": "Text", "value" :"CAL"} - } + "entity_type": "Lamp", + "resource": "/iot/d", + "protocol": "PDI-IoTA-UltraLight", + "commands": [ + { "name": "on", "type": "command" }, + { "name": "off", "type": "command" } + ], + "attributes": [ + { "object_id": "s", "name": "state", "type": "Text" }, + { + "object_id": "l", + "name": "luminosity", + "type": "Integer", + "metadata": { + "unitCode": { "type": "Text", "value": "CAL" } + } } - ], - "static_attributes": [ - {"name": "category", "type":"Text", "value": ["actuator","sensor"]}, - {"name": "controlledProperty", "type": "Text", "value": ["light"], - "metadata":{ - "includes":{"type": "Text", "value" :["state", "luminosity"]}, - "alias":{"type": "Text", "value" :"lamp"} + ], + "static_attributes": [ + { "name": "category", "type": "Text", "value": ["actuator", "sensor"] }, + { + "name": "controlledProperty", + "type": "Text", + "value": ["light"], + "metadata": { + "includes": { "type": "Text", "value": ["state", "luminosity"] }, + "alias": { "type": "Text", "value": "lamp" } } - }, - ] - } + } + ] +} ``` Metadata could also has `expression` like attributes in order to expand it: @@ -320,35 +325,37 @@ e.g.: ```json { - "entity_type": "Lamp", - "resource": "/iot/d", - "protocol": "PDI-IoTA-UltraLight", -..etc - "commands": [ - {"name": "on","type": "command"}, - {"name": "off","type": "command"} - ], - "attributes": [ - {"object_id": "s", "name": "state", "type":"Text"}, - {"object_id": "l", "name": "luminosity", "type":"Integer", - "metadata":{ - "unitCode":{"type": "Text", "value" :"CAL"} - } + "entity_type": "Lamp", + "resource": "/iot/d", + "protocol": "PDI-IoTA-UltraLight", + "commands": [ + { "name": "on", "type": "command" }, + { "name": "off", "type": "command" } + ], + "attributes": [ + { "object_id": "s", "name": "state", "type": "Text" }, + { + "object_id": "l", + "name": "luminosity", + "type": "Integer", + "metadata": { + "unitCode": { "type": "Text", "value": "CAL" } + } } - ], - "static_attributes": [ - {"name": "category", "type":"Text", "value": ["actuator","sensor"]}, - {"name": "controlledProperty", "type": "Text", "value": ["light"], - "metadata":{ - "includes":{"type": "Text", - "value" :["state", "luminosity"], - "expression": "level / 100" - }, - "alias":{"type": "Text", "value" :"lamp"} + ], + "static_attributes": [ + { "name": "category", "type": "Text", "value": ["actuator", "sensor"] }, + { + "name": "controlledProperty", + "type": "Text", + "value": ["light"], + "metadata": { + "includes": { "type": "Text", "value": ["state", "luminosity"], "expression": "level / 100" }, + "alias": { "type": "Text", "value": "lamp" } } - }, - ] - } + } + ] +} ``` ### NGSI-LD data and metadata considerations From a6f7daf74a00d88a8c4a189fbe608897e1377da5 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 20 May 2024 11:02:46 +0200 Subject: [PATCH 61/83] update doc --- doc/api.md | 122 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/doc/api.md b/doc/api.md index d661861c0..f3e285f75 100644 --- a/doc/api.md +++ b/doc/api.md @@ -32,6 +32,7 @@ - [Measurement transformation order](#measurement-transformation-order) - [Multientity measurement transformation support (`object_id`)](#multientity-measurement-transformation-support-object_id) - [Timestamp Processing](#timestamp-processing) + - [Multimeasure support](#multimeasure-support) - [Overriding global Context Broker host](#overriding-global-context-broker-host) - [Multitenancy, FIWARE Service and FIWARE ServicePath](#multitenancy-fiware-service-and-fiware-servicepath) - [Secured access to the Context Broker](#secured-access-to-the-context-broker) @@ -1044,6 +1045,127 @@ Some additional considerations to take into account: measure of after a mapping, as described in the previous bullet) then it is refused (so a failover to server timestamp will take place). +## Multimeasure support + +A device could receive several measures at the same time. + +For example: + +```json +[ + { + "vol": 0 + }, + { + "vol": 1 + }, + { + "vol": 2 + } +] +``` + +In this case a batch update (`/op/v2/update`) to CB will be generated with the following NGSI v2 payload: + +```json +{ + "actionType": "append", + "entities": [ + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 0 + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 1 + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 1 + } + } + ] +} +``` + +Moreover if a multimeasure contains TimeInstant attribute, then CB update is sorted by attribute TimeInstant: + +For example: + +```json +[ + { + "vol": 0, + "TimeInstant": "2024-04-10T10:15:00Z" + }, + { + "vol": 1, + "TimeInstant": "2024-04-10T10:10:00Z" + }, + { + "vol": 2, + "TimeInstant": "2024-04-10T10:05:00Z" + } +] +``` + +In this case a batch update (`/op/v2/update`) to CB will be generated with the following NGSI v2 payload: + +```json +{ + "actionType": "append", + "entities": [ + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 2 + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-04-10T10:05:00Z" + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 1 + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-04-10T10:10:00Z" + } + }, + { + "id": "ws", + "type": "WeatherStation", + "vol": { + "type": "Number", + "value": 0 + }, + "TimeInstant": { + "type": "DateTime", + "value": "2024-04-10T10:15:00Z" + } + } + ] +} +``` + ## Overriding global Context Broker host **cbHost**: Context Broker host URL. This option can be used to override the global CB configuration for specific types From 9e4ea596cd25ca90e0793b967e1aa34919eb6fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Mon, 20 May 2024 17:16:35 +0200 Subject: [PATCH 62/83] Apply suggestions from code review --- doc/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api.md b/doc/api.md index f3e285f75..105654150 100644 --- a/doc/api.md +++ b/doc/api.md @@ -1065,7 +1065,7 @@ For example: ] ``` -In this case a batch update (`/op/v2/update`) to CB will be generated with the following NGSI v2 payload: +In this case a batch update (`POST /v2/op/update`) to CB will be generated with the following NGSI v2 payload: ```json { From 15a82ebbd7996d6e30d213e41524c1754dfd7e11 Mon Sep 17 00:00:00 2001 From: mapedraza <40356341+mapedraza@users.noreply.github.com> Date: Mon, 20 May 2024 17:53:35 +0200 Subject: [PATCH 63/83] Modify multimeasures doc --- test/functional/README.md | 97 ++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 37 deletions(-) diff --git a/test/functional/README.md b/test/functional/README.md index 47d477be0..6020a6952 100644 --- a/test/functional/README.md +++ b/test/functional/README.md @@ -39,7 +39,7 @@ test cases are automatically generated. Each test case is defined as an object w or if the `transport` element is not defined. See the "Advanced features" section for more information. - `shouldName`: The name of the `IT` test case. This will be used to generate the test case name in the mocha test suite. - - `type`: The type of the test case. This can be `single` or `multientity`. See the "Advanced features" section + - `type`: The type of the test case. This can be `single`, `multimeasure` or `multientity`. See the "Advanced features" section for more information. - `measure`: The JSON object that will be sent to the IoTA JSON measure API. This will be used to send the measure. It contains the following elements: @@ -199,41 +199,11 @@ as a batch operation (see the following example). #### Multimeasures -It is also supported to test cases in which is sent more than one measure. To do so, you need to define the test case -`expectation` as an array, with one object for each measurement. Then, the suite will recognize the array length and will -expect the same number of NGSI requests. I.E: +It is also supported to test cases in which is sent more than one measure. To do so, you need to set add to the test case +the parameter `should.type` to the value `'multimeasure'`. -```js -[ - { - id: 'TheLightType2:MQTT_2', - type: 'TheLightType2', - temperature: { - value: 10, - type: 'Number' - }, - status: { - value: false, - type: 'Boolean' - } - }, - { - id: 'TheLightType2:MQTT_2', - type: 'TheLightType2', - temperature: { - value: 20, - type: 'Number' - }, - status: { - value: true, - type: 'Boolean' - } - } -]; -``` - -You also should define the measure as multimeasure. This is done by defining the `measure` JSON element as an array of -objects. Each object will be a measure that will be sent to the Context Broker in a different request. I.E: +You must define the measure as multimeasure. This is done by defining the `measure` JSON element as an array of +objects. I.E: ```javascript measure: { @@ -246,16 +216,69 @@ measure: { json: [ { s: false, - t: 10 + t: 21 }, { s: true, - t: 20 + t: 22 + }, + { + s: false, + t: 23 + } + ] +} +``` + +And you should define the test case `expectation` as an object, following a Context Broker batch operation. I.E: + +```js +expectation: { + actionType: 'append', + entities: [ + { + id: 'TheLightType2:MQTT_2', + type: 'TheLightType2', + temperature: { + type: 'Number', + value: 21 + }, + status: { + type: 'Boolean', + value: false + } + }, + { + id: 'TheLightType2:MQTT_2', + type: 'TheLightType2', + temperature: { + type: 'Number', + value: 22 + }, + status: { + type: 'Boolean', + value: true + } + }, + { + id: 'TheLightType2:MQTT_2', + type: 'TheLightType2', + temperature: { + type: 'Number', + value: 23 + }, + status: { + type: 'Boolean', + value: false + } } ] } ``` +Then, a batch request would be sent to the Context Broker containing the different measures. More information about +how the IoT Agent send multimeasures to the Context Broker [here](/doc/api.md#multimeasure-support). + #### Transport The test suite supports using the internal node lib function `iotAgentLib.update`, `HTTP` or `MQTT` for measure sending. From 6d8642bb0a3cd8104ab4a9d738ad3ef085bf8b7c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 21 May 2024 08:00:53 +0200 Subject: [PATCH 64/83] Update doc/api.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- doc/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api.md b/doc/api.md index 105654150..17500ba85 100644 --- a/doc/api.md +++ b/doc/api.md @@ -1120,7 +1120,7 @@ For example: ] ``` -In this case a batch update (`/op/v2/update`) to CB will be generated with the following NGSI v2 payload: +In this case a batch update (`POST /v2/op/update`) to CB will be generated with the following NGSI v2 payload: ```json { From 28fb02fcb1e986749b26eb04e5f0979485711c58 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 21 May 2024 10:35:30 +0200 Subject: [PATCH 65/83] use oldDevice --- lib/services/devices/deviceService.js | 1 + lib/services/northBound/deviceProvisioningServer.js | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/services/devices/deviceService.js b/lib/services/devices/deviceService.js index 495643d36..8ad982596 100644 --- a/lib/services/devices/deviceService.js +++ b/lib/services/devices/deviceService.js @@ -518,6 +518,7 @@ function getDevice(deviceId, apikey, service, subservice, callback) { * @param {String} device JSON object contain the device to update. */ function updateDevice(device, callback) { + logger.debug(context, 'updateDevice %j', device); config.getRegistry().update(device, callback); } diff --git a/lib/services/northBound/deviceProvisioningServer.js b/lib/services/northBound/deviceProvisioningServer.js index c9b3ccb07..c2285d508 100644 --- a/lib/services/northBound/deviceProvisioningServer.js +++ b/lib/services/northBound/deviceProvisioningServer.js @@ -398,11 +398,11 @@ function handleRemoveDevices(req, res, next) { * This middleware handles updates in the provisioning devices. The only attribute */ function handleUpdateDevice(req, res, next) { - function applyUpdatingHandler(device, callback) { + function applyUpdatingHandler(newDevice, oldDevice, callback) { if (updatingHandler) { - updatingHandler(device, callback); + updatingHandler(newDevice, oldDevice, callback); } else { - callback(null, device); + callback(null, newdevice); } } @@ -429,7 +429,7 @@ function handleUpdateDevice(req, res, next) { isTypeOrNameUpdated = true; } async.waterfall( - [apply(applyUpdatingHandler, newDevice)], + [apply(applyUpdatingHandler, newDevice, device)], function handleUpdating(error, newDeviceUpdated) { deviceService.updateRegister( newDeviceUpdated, From e6272921a46bcb171f38e7733c03f4c5c9afc835 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 21 May 2024 11:06:16 +0200 Subject: [PATCH 66/83] use previousDevice values --- lib/services/devices/devices-NGSI-v2.js | 6 +++--- lib/services/northBound/deviceProvisioningServer.js | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/services/devices/devices-NGSI-v2.js b/lib/services/devices/devices-NGSI-v2.js index 77ee7431f..242789253 100644 --- a/lib/services/devices/devices-NGSI-v2.js +++ b/lib/services/devices/devices-NGSI-v2.js @@ -247,7 +247,7 @@ function updateEntityNgsi2(deviceData, updatedDevice, callback) { * * @param {Object} deviceObj Object with all the device information (mandatory). */ -function updateRegisterDeviceNgsi2(deviceObj, entityInfoUpdated, callback) { +function updateRegisterDeviceNgsi2(deviceObj, previousDevice, entityInfoUpdated, callback) { if (!deviceObj.id || !deviceObj.type) { callback(new errors.MissingAttributes('Id or device missing', deviceObj)); return; @@ -352,7 +352,7 @@ function updateRegisterDeviceNgsi2(deviceObj, entityInfoUpdated, callback) { apply( config.getRegistry().get, deviceObj.id, - deviceObj.apikey, + previousDevice.apikey, // it could be updated deviceObj.service, deviceObj.subservice ), @@ -371,7 +371,7 @@ function updateRegisterDeviceNgsi2(deviceObj, entityInfoUpdated, callback) { apply( config.getRegistry().get, deviceObj.id, - deviceObj.apikey, + previousDevice.apikey, deviceObj.service, deviceObj.subservice ), diff --git a/lib/services/northBound/deviceProvisioningServer.js b/lib/services/northBound/deviceProvisioningServer.js index c2285d508..9de2053b1 100644 --- a/lib/services/northBound/deviceProvisioningServer.js +++ b/lib/services/northBound/deviceProvisioningServer.js @@ -433,6 +433,7 @@ function handleUpdateDevice(req, res, next) { function handleUpdating(error, newDeviceUpdated) { deviceService.updateRegister( newDeviceUpdated, + device, isTypeOrNameUpdated, function handleDeviceUpdate(error) { if (error) { From d98aeba8370055b020f00fb91b6ce96a7d416e7f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 21 May 2024 13:08:59 +0200 Subject: [PATCH 67/83] fix typo --- lib/services/devices/deviceService.js | 4 ++-- lib/services/devices/devices-NGSI-LD.js | 6 +++--- lib/services/devices/devices-NGSI-mixed.js | 6 +++--- lib/services/northBound/deviceProvisioningServer.js | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/services/devices/deviceService.js b/lib/services/devices/deviceService.js index 8ad982596..0c2dc78c1 100644 --- a/lib/services/devices/deviceService.js +++ b/lib/services/devices/deviceService.js @@ -444,8 +444,8 @@ function unregisterDevice(id, apikey, service, subservice, callback) { }); } -function updateRegisterDevice(deviceObj, entityInfoUpdated, callback) { - deviceHandler.updateRegisterDevice(deviceObj, entityInfoUpdated, callback); +function updateRegisterDevice(deviceObj, previousDevice, entityInfoUpdated, callback) { + deviceHandler.updateRegisterDevice(deviceObj, previousDevice, entityInfoUpdated, callback); } /** diff --git a/lib/services/devices/devices-NGSI-LD.js b/lib/services/devices/devices-NGSI-LD.js index 42487e6fb..e83401e30 100644 --- a/lib/services/devices/devices-NGSI-LD.js +++ b/lib/services/devices/devices-NGSI-LD.js @@ -182,7 +182,7 @@ function updateEntityNgsiLD(deviceData, updatedDevice, callback) { * * @param {Object} deviceObj Object with all the device information (mandatory). */ -function updateRegisterDeviceNgsiLD(deviceObj, entityInfoUpdated, callback) { +function updateRegisterDeviceNgsiLD(deviceObj, previousDevice, entityInfoUpdated, callback) { if (!deviceObj.id || !deviceObj.type) { callback(new errors.MissingAttributes('Id or device missing', deviceObj)); return; @@ -280,7 +280,7 @@ function updateRegisterDeviceNgsiLD(deviceObj, entityInfoUpdated, callback) { apply( config.getRegistry().get, deviceObj.id, - deviceObj.apikey, + previousDevice.apikey, deviceObj.service, deviceObj.subservice ), @@ -299,7 +299,7 @@ function updateRegisterDeviceNgsiLD(deviceObj, entityInfoUpdated, callback) { apply( config.getRegistry().get, deviceObj.id, - deviceObj.apikey, + previousDevice.apikey, deviceObj.service, deviceObj.subservice ), diff --git a/lib/services/devices/devices-NGSI-mixed.js b/lib/services/devices/devices-NGSI-mixed.js index e257f638d..0dbd3f966 100644 --- a/lib/services/devices/devices-NGSI-mixed.js +++ b/lib/services/devices/devices-NGSI-mixed.js @@ -38,11 +38,11 @@ const deviceHandlerV2 = require('./devices-NGSI-v2'); * * @param {Object} deviceObj Object with all the device information (mandatory). */ -function updateRegisterDeviceNgsiMixed(deviceObj, entityInfoUpdated, callback) { +function updateRegisterDeviceNgsiMixed(deviceObj, previousDevice, entityInfoUpdated, callback) { if (config.checkNgsiLD(deviceObj)) { - deviceHandlerLD.updateRegisterDevice(deviceObj, entityInfoUpdated, callback); + deviceHandlerLD.updateRegisterDevice(deviceObj, previousDevice, entityInfoUpdated, callback); } else { - deviceHandlerV2.updateRegisterDevice(deviceObj, entityInfoUpdated, callback); + deviceHandlerV2.updateRegisterDevice(deviceObj, previousDevice, entityInfoUpdated, callback); } } diff --git a/lib/services/northBound/deviceProvisioningServer.js b/lib/services/northBound/deviceProvisioningServer.js index 9de2053b1..510b9802f 100644 --- a/lib/services/northBound/deviceProvisioningServer.js +++ b/lib/services/northBound/deviceProvisioningServer.js @@ -402,7 +402,7 @@ function handleUpdateDevice(req, res, next) { if (updatingHandler) { updatingHandler(newDevice, oldDevice, callback); } else { - callback(null, newdevice); + callback(null, newDevice); } } From 35d7d0bd4a2772058a64d625f3df9cc8aad5bc10 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 22 May 2024 10:12:11 +0200 Subject: [PATCH 68/83] update updateRegsiter calls --- .../device-update-registration_test.js | 10 +++++----- .../device-update-registration_test.js | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/unit/ngsi-ld/provisioning/device-update-registration_test.js b/test/unit/ngsi-ld/provisioning/device-update-registration_test.js index 912943855..3c39b33fa 100644 --- a/test/unit/ngsi-ld/provisioning/device-update-registration_test.js +++ b/test/unit/ngsi-ld/provisioning/device-update-registration_test.js @@ -186,14 +186,14 @@ describe('NGSI-LD - IoT Agent Device Update Registration', function () { }); it('should register as ContextProvider of its lazy attributes', function (done) { - iotAgentLib.updateRegister(deviceUpdated, false, function (error) { + iotAgentLib.updateRegister(deviceUpdated, device1, false, function (error) { should.not.exist(error); contextBrokerMock.done(); done(); }); }); it('should store the new values in the registry', function (done) { - iotAgentLib.updateRegister(deviceUpdated, false, function (error, data) { + iotAgentLib.updateRegister(deviceUpdated, device1, false, function (error, data) { iotAgentLib.getDevice(deviceUpdated.id, null, 'smartgondor', 'gardens', function (error, deviceResult) { should.not.exist(error); should.exist(deviceResult); @@ -236,7 +236,7 @@ describe('NGSI-LD - IoT Agent Device Update Registration', function () { // }); // }); it('should store the new values in the registry', function (done) { - iotAgentLib.updateRegister(deviceCommandUpdated, false, function (error, data) { + iotAgentLib.updateRegister(deviceCommandUpdated, device1, false, function (error, data) { iotAgentLib.getDevice( deviceCommandUpdated.id, null, @@ -257,7 +257,7 @@ describe('NGSI-LD - IoT Agent Device Update Registration', function () { describe('When a update action is executed in a non registered device', function () { it('should return a DEVICE_NOT_FOUND error', function (done) { - iotAgentLib.updateRegister(unknownDevice, false, function (error) { + iotAgentLib.updateRegister(unknownDevice, device1, false, function (error) { should.exist(error); error.name.should.equal('DEVICE_NOT_FOUND'); done(); @@ -273,7 +273,7 @@ describe('NGSI-LD - IoT Agent Device Update Registration', function () { }); it('should return a REGISTRATION_ERROR error in the update action', function (done) { - iotAgentLib.updateRegister(deviceUpdated, false, function (error) { + iotAgentLib.updateRegister(deviceUpdated, device1, false, function (error) { should.exist(error); //error.name.should.equal('UNREGISTRATION_ERROR'); done(); diff --git a/test/unit/ngsiv2/provisioning/device-update-registration_test.js b/test/unit/ngsiv2/provisioning/device-update-registration_test.js index 5b1576ac2..d550ae715 100644 --- a/test/unit/ngsiv2/provisioning/device-update-registration_test.js +++ b/test/unit/ngsiv2/provisioning/device-update-registration_test.js @@ -138,7 +138,7 @@ const unknownDevice = { describe('NGSI-v2 - IoT Agent Device Update Registration', function () { beforeEach(function (done) { delete device1.registrationId; - logger.setLevel('FATAL'); + logger.setLevel('DEBUG'); nock.cleanAll(); @@ -197,7 +197,7 @@ describe('NGSI-v2 - IoT Agent Device Update Registration', function () { }); it('should register as ContextProvider of its lazy attributes', function (done) { - iotAgentLib.updateRegister(deviceUpdated, false, function (error) { + iotAgentLib.updateRegister(deviceUpdated, device1, false, function (error) { should.not.exist(error); contextBrokerMock.done(); done(); @@ -205,7 +205,7 @@ describe('NGSI-v2 - IoT Agent Device Update Registration', function () { }); it('should store the new values in the registry', function (done) { - iotAgentLib.updateRegister(deviceUpdated, false, function (error, data) { + iotAgentLib.updateRegister(deviceUpdated, device1, false, function (error, data) { iotAgentLib.getDevice(deviceUpdated.id, null, 'smartgondor', 'gardens', function (error, deviceResult) { should.not.exist(error); should.exist(deviceResult); @@ -253,7 +253,7 @@ describe('NGSI-v2 - IoT Agent Device Update Registration', function () { }); it('should register as ContextProvider of its commands and create the additional attributes', function (done) { - iotAgentLib.updateRegister(deviceCommandUpdated, false, function (error) { + iotAgentLib.updateRegister(deviceCommandUpdated, device1, false, function (error) { should.not.exist(error); contextBrokerMock.done(); done(); @@ -261,7 +261,7 @@ describe('NGSI-v2 - IoT Agent Device Update Registration', function () { }); it('should store the new values in the registry', function (done) { - iotAgentLib.updateRegister(deviceCommandUpdated, false, function (error, data) { + iotAgentLib.updateRegister(deviceCommandUpdated, device1, false, function (error, data) { iotAgentLib.getDevice( deviceCommandUpdated.id, null, @@ -282,7 +282,7 @@ describe('NGSI-v2 - IoT Agent Device Update Registration', function () { describe('When a update action is executed in a non registered device', function () { it('should return a DEVICE_NOT_FOUND error', function (done) { - iotAgentLib.updateRegister(unknownDevice, false, function (error) { + iotAgentLib.updateRegister(unknownDevice, device1, false, function (error) { should.exist(error); error.name.should.equal('DEVICE_NOT_FOUND'); done(); @@ -304,7 +304,7 @@ describe('NGSI-v2 - IoT Agent Device Update Registration', function () { }); it('should return a REGISTRATION_ERROR error in the update action', function (done) { - iotAgentLib.updateRegister(deviceUpdated, false, function (error) { + iotAgentLib.updateRegister(deviceUpdated, device1, false, function (error) { should.exist(error); error.name.should.equal('UNREGISTRATION_ERROR'); done(); From b39b48e0dd0111e70042f4a635cbba909e7fa68c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Wed, 22 May 2024 16:20:18 +0200 Subject: [PATCH 69/83] update using previous apikey --- lib/services/devices/deviceRegistryMemory.js | 2 +- lib/services/devices/deviceRegistryMongoDB.js | 4 ++-- lib/services/devices/deviceService.js | 2 +- lib/services/devices/devices-NGSI-LD.js | 4 ++-- lib/services/devices/devices-NGSI-v2.js | 4 ++-- lib/services/ngsi/subscription-NGSI-LD.js | 4 ++-- lib/services/ngsi/subscription-NGSI-v2.js | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/services/devices/deviceRegistryMemory.js b/lib/services/devices/deviceRegistryMemory.js index b801f2744..f191692a2 100644 --- a/lib/services/devices/deviceRegistryMemory.js +++ b/lib/services/devices/deviceRegistryMemory.js @@ -176,7 +176,7 @@ function getByName(name, service, subservice, callback) { getByNameAndType(name, null, service, subservice, callback); } -function update(device, callback) { +function update(previousDevice, device, callback) { registeredDevices[device.service][device.id] = deepClone(device); callback(null, device); } diff --git a/lib/services/devices/deviceRegistryMongoDB.js b/lib/services/devices/deviceRegistryMongoDB.js index bddac06df..d966a4c62 100644 --- a/lib/services/devices/deviceRegistryMongoDB.js +++ b/lib/services/devices/deviceRegistryMongoDB.js @@ -290,9 +290,9 @@ function getByName(name, service, servicepath, callback) { * * @param {Object} device Device object with the new values to write. */ -function update(device, callback) { +function update(previousDevice, device, callback) { logger.debug(context, 'Storing updated values for device [%s]:\n%s', device.id, JSON.stringify(device, null, 4)); - getDevice(device.id, device.apikey, device.service, device.subservice, function (error, data) { + getDevice(device.id, previousDevice.apikey, device.service, device.subservice, function (error, data) { if (error) { callback(error); } else { diff --git a/lib/services/devices/deviceService.js b/lib/services/devices/deviceService.js index 0c2dc78c1..03be2722c 100644 --- a/lib/services/devices/deviceService.js +++ b/lib/services/devices/deviceService.js @@ -519,7 +519,7 @@ function getDevice(deviceId, apikey, service, subservice, callback) { */ function updateDevice(device, callback) { logger.debug(context, 'updateDevice %j', device); - config.getRegistry().update(device, callback); + config.getRegistry().update(device, device, callback); } /** diff --git a/lib/services/devices/devices-NGSI-LD.js b/lib/services/devices/devices-NGSI-LD.js index e83401e30..b9c050f75 100644 --- a/lib/services/devices/devices-NGSI-LD.js +++ b/lib/services/devices/devices-NGSI-LD.js @@ -289,7 +289,7 @@ function updateRegisterDeviceNgsiLD(deviceObj, previousDevice, entityInfoUpdated apply(combineWithNewDevice, deviceObj), apply(registrationUtils.sendRegistrations, false), apply(registrationUtils.processContextRegistration, deviceObj), - config.getRegistry().update + apply(config.getRegistry().update, previousDevice) ], callback ); @@ -308,7 +308,7 @@ function updateRegisterDeviceNgsiLD(deviceObj, previousDevice, entityInfoUpdated apply(combineWithNewDevice, deviceObj), apply(registrationUtils.sendRegistrations, false), apply(registrationUtils.processContextRegistration, deviceObj), - config.getRegistry().update + apply(config.getRegistry().update, previousDevice) ], callback ); diff --git a/lib/services/devices/devices-NGSI-v2.js b/lib/services/devices/devices-NGSI-v2.js index 242789253..f5942467f 100644 --- a/lib/services/devices/devices-NGSI-v2.js +++ b/lib/services/devices/devices-NGSI-v2.js @@ -361,7 +361,7 @@ function updateRegisterDeviceNgsi2(deviceObj, previousDevice, entityInfoUpdated, apply(combineWithNewDevice, deviceObj), apply(registrationUtils.sendRegistrations, false), apply(registrationUtils.processContextRegistration, deviceObj), - config.getRegistry().update + apply(config.getRegistry().update, previousDevice) ], callback ); @@ -380,7 +380,7 @@ function updateRegisterDeviceNgsi2(deviceObj, previousDevice, entityInfoUpdated, apply(combineWithNewDevice, deviceObj), apply(registrationUtils.sendRegistrations, false), apply(registrationUtils.processContextRegistration, deviceObj), - config.getRegistry().update + apply(config.getRegistry().update, previousDevice) ], callback ); diff --git a/lib/services/ngsi/subscription-NGSI-LD.js b/lib/services/ngsi/subscription-NGSI-LD.js index 4eb0f2129..f5ac6384a 100644 --- a/lib/services/ngsi/subscription-NGSI-LD.js +++ b/lib/services/ngsi/subscription-NGSI-LD.js @@ -89,7 +89,7 @@ function createSubscriptionHandlerNgsiLD(device, triggers, store, callback) { triggers }); - config.getRegistry().update(device, callback); + config.getRegistry().update(device, device, callback); } else { callback(null, response.headers.location); } @@ -203,7 +203,7 @@ function createUnsubscribeHandlerNgsiLD(device, id, callback) { callback(new errors.BadRequest(body.orionError.details)); } else { device.subscriptions.splice(device.subscriptions.indexOf(id), 1); - config.getRegistry().update(device, callback); + config.getRegistry().update(device, device, callback); } }; } diff --git a/lib/services/ngsi/subscription-NGSI-v2.js b/lib/services/ngsi/subscription-NGSI-v2.js index ccaf64f0c..7b0be9ccb 100644 --- a/lib/services/ngsi/subscription-NGSI-v2.js +++ b/lib/services/ngsi/subscription-NGSI-v2.js @@ -91,7 +91,7 @@ function createSubscriptionHandlerNgsi2(device, triggers, store, callback) { triggers }); - config.getRegistry().update(device, callback); + config.getRegistry().update(device, device, callback); } else { callback(null, response.headers.location); } @@ -203,7 +203,7 @@ function createUnsubscribeHandlerNgsi2(device, id, callback) { callback(new errors.BadRequest(body.orionError.details)); } else { device.subscriptions.splice(device.subscriptions.indexOf(id), 1); - config.getRegistry().update(device, callback); + config.getRegistry().update(device, device, callback); } }; } From 35b9b531a4a265391606d736e4b88c8c8f4a6e9f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 23 May 2024 14:31:25 +0200 Subject: [PATCH 70/83] update CNR --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 25e0be54d..bea908283 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1 +1,2 @@ +- Fix: update device using previous device apikey to avoid error when apikey is updated - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (telefonicaid/iotagent-json#827) From a32a5896246c4ecd2601e6f093bb17391036fa0e Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 23 May 2024 14:32:16 +0200 Subject: [PATCH 71/83] Update CNR --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index bea908283..a9455874a 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,2 @@ -- Fix: update device using previous device apikey to avoid error when apikey is updated +- Fix: update device using previous device apikey to avoid error when apikey is updated /(iota-json#833) - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (telefonicaid/iotagent-json#827) From bba857be99a313e0dbec481b04481350d29014a2 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 23 May 2024 15:24:12 +0200 Subject: [PATCH 72/83] Update device-update-registration_test.js --- .../unit/ngsiv2/provisioning/device-update-registration_test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/ngsiv2/provisioning/device-update-registration_test.js b/test/unit/ngsiv2/provisioning/device-update-registration_test.js index d550ae715..d97efee81 100644 --- a/test/unit/ngsiv2/provisioning/device-update-registration_test.js +++ b/test/unit/ngsiv2/provisioning/device-update-registration_test.js @@ -138,7 +138,7 @@ const unknownDevice = { describe('NGSI-v2 - IoT Agent Device Update Registration', function () { beforeEach(function (done) { delete device1.registrationId; - logger.setLevel('DEBUG'); + logger.setLevel('FATAL'); nock.cleanAll(); From 6c2dc8daaf6c79dd9fce89f32d47a589b76ace66 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Mon, 27 May 2024 12:35:33 +0200 Subject: [PATCH 73/83] add update device apikey test --- .../updateProvisionDeviceWithApikey.json | 4 +++ .../updateProvisionedDevices-test.js | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 test/unit/examples/deviceProvisioningRequests/updateProvisionDeviceWithApikey.json diff --git a/test/unit/examples/deviceProvisioningRequests/updateProvisionDeviceWithApikey.json b/test/unit/examples/deviceProvisioningRequests/updateProvisionDeviceWithApikey.json new file mode 100644 index 000000000..4b9b51e29 --- /dev/null +++ b/test/unit/examples/deviceProvisioningRequests/updateProvisionDeviceWithApikey.json @@ -0,0 +1,4 @@ +{ + "apikey": "APIKEY2", + "timezone": "Europe/Madrid" +} diff --git a/test/unit/ngsiv2/provisioning/updateProvisionedDevices-test.js b/test/unit/ngsiv2/provisioning/updateProvisionedDevices-test.js index 4ea3b2239..81974beaf 100644 --- a/test/unit/ngsiv2/provisioning/updateProvisionedDevices-test.js +++ b/test/unit/ngsiv2/provisioning/updateProvisionedDevices-test.js @@ -289,6 +289,41 @@ describe('NGSI-v2 - Device provisioning API: Update provisioned devices', functi }); }); }); + describe('When an update request arrives with a new Apikey', function () { + const optionsUpdate = { + url: 'http://localhost:' + iotAgentConfig.server.port + '/iot/devices/Light1', + method: 'PUT', + headers: { + 'fiware-service': 'smartgondor', + 'fiware-servicepath': '/gardens' + }, + json: utils.readExampleFile( + './test/unit/examples/deviceProvisioningRequests/updateProvisionDeviceWithApikey.json' + ) + }; + + beforeEach(function () { + contextBrokerMock + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', '/gardens') + .delete('/v2/registrations/6319a7f5254b05844116584d', '') + .reply(204); + + contextBrokerMock + .matchHeader('fiware-service', 'smartgondor') + .matchHeader('fiware-servicepath', '/gardens') + .post('/v2/registrations') + .reply(201, null, { Location: '/v2/registrations/4419a7f5254b058441165849' }); + }); + + it('should raise a 204 error', function (done) { + request(optionsUpdate, function (error, response, body) { + should.not.exist(error); + response.statusCode.should.equal(204); + done(); + }); + }); + }); describe('When a wrong update request payload arrives', function () { const optionsUpdate = { url: 'http://localhost:' + iotAgentConfig.server.port + '/iot/devices/Light1', From 053e9165a5667cc29989732018e6ac308a212dc5 Mon Sep 17 00:00:00 2001 From: sgheppy Date: Fri, 31 May 2024 08:48:21 +0200 Subject: [PATCH 74/83] fix ENTITY_GENERIC_ERROR when context broker respond with status code in update operation with orion ld 1.5.1 during the transmission of the first measure operation the context broker respond with 201 status code --- lib/services/ngsi/entities-NGSI-LD.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/services/ngsi/entities-NGSI-LD.js b/lib/services/ngsi/entities-NGSI-LD.js index be8512905..5b6232574 100644 --- a/lib/services/ngsi/entities-NGSI-LD.js +++ b/lib/services/ngsi/entities-NGSI-LD.js @@ -341,7 +341,7 @@ function generateNGSILDOperationHandler(operationName, entityName, typeInformati } else if ( response && operationName === 'update' && - (response.statusCode === 200 || response.statusCode === 204) + (response.statusCode === 200 || response.statusCode === 204 || response.statusCode === 201) ) { logger.info(context, 'Received the following response from the CB: Value updated successfully\n'); alarms.release(constants.ORION_ALARM); From 31b46f44214e4049f57cfeb7de1e0be2dcac269f Mon Sep 17 00:00:00 2001 From: aquarta <163529657+aquarta@users.noreply.github.com> Date: Fri, 31 May 2024 14:24:05 +0200 Subject: [PATCH 75/83] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 05feb0873..6949f2d49 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ - Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825, #1612) - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) +- Fix: accept 201 status code from context broker with NGSI-LD interface when first measure is sent and device is created in it (#1617) From 8a34ce39b6e1ad27294bd9249f586a911285f766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Wed, 5 Jun 2024 09:35:54 +0200 Subject: [PATCH 76/83] Update CHANGES_NEXT_RELEASE --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index b0572a5f5..d04285253 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Fix: update device using previous device apikey to avoid error when apikey is updated /(iota-json#833) +- Fix: update device using previous device apikey to avoid error when apikey is updated (iota-json#833) - Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825, #1612) - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) - Fix: accept 201 status code from context broker with NGSI-LD interface when first measure is sent and device is created in it (#1617) From 9ea5d39af63a8f0e23f5f87fbaf0834643017a97 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 6 Jun 2024 10:39:13 +0200 Subject: [PATCH 77/83] check explicitAttrs array is not empty before add timestamp as attribute --- lib/services/ngsi/entities-NGSI-v2.js | 2 +- test/functional/testCases.js | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/lib/services/ngsi/entities-NGSI-v2.js b/lib/services/ngsi/entities-NGSI-v2.js index dc5e3b733..a02d9aa60 100644 --- a/lib/services/ngsi/entities-NGSI-v2.js +++ b/lib/services/ngsi/entities-NGSI-v2.js @@ -504,7 +504,7 @@ function sendUpdateValueNgsi2(entityName, measures, typeInformation, token, call if (typeof typeInformation.explicitAttrs === 'string') { try { explicit = jexlParser.applyExpression(typeInformation.explicitAttrs, jexlctxt, typeInformation); - if (explicit instanceof Array && mustInsertTimeInstant) { + if (explicit instanceof Array && explicit.length > 0 && mustInsertTimeInstant) { explicit.push(constants.TIMESTAMP_ATTRIBUTE); } logger.debug( diff --git a/test/functional/testCases.js b/test/functional/testCases.js index ab5cfd4c5..389d6f8ac 100644 --- a/test/functional/testCases.js +++ b/test/functional/testCases.js @@ -3045,14 +3045,7 @@ const testCases = [ t: '2015-12-14T08:06:01.468Z' } }, - expectation: { - id: 'TestType:TestDevice', - type: 'TestType', - TimeInstant: { - type: 'DateTime', - value: '2015-12-14T08:06:01.468Z' - } - } + expectation: [] }, { shouldName: From 3176f6cf595a88350a0b25ecd1097fb14c2fe18f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 6 Jun 2024 11:24:24 +0200 Subject: [PATCH 78/83] update CNR --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index d04285253..5169f122e 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ +- Feature: do not propage TimeInstnat when explicitAttrs is empty array - Fix: update device using previous device apikey to avoid error when apikey is updated (iota-json#833) - Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825, #1612) - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) From 37c255e8748b033c3aeb991e78c18c5f3dae81b1 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 6 Jun 2024 12:53:56 +0200 Subject: [PATCH 79/83] update CNR --- doc/api.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/api.md b/doc/api.md index b614a2799..f2b74c59b 100644 --- a/doc/api.md +++ b/doc/api.md @@ -464,7 +464,8 @@ element. By adding the field `explicitAttrs` with `true` value to device or conf measure elements that are not defined in the mappings of device or config group, persisting only the one defined in the mappings of the provision. If `explicitAttrs` is provided both at device and config group level, the device level takes precedence. Additionally `explicitAttrs` can be used to define which measures (identified by their attribute names, not -by their object_id) defined in JSON/JEXL array will be propagated to NGSI interface. +by their object_id) defined in JSON/JEXL array will be propagated to NGSI interface. When `explicitAttrs` is an array or +and JEXL expression resulting in to Array, if this array is empty then `TimeInstant` is not propaged to CB. The different possibilities are summarized below: From b0422bd9d01751565b42153dff0948f0ac2ebf3c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 6 Jun 2024 12:56:40 +0200 Subject: [PATCH 80/83] Update CHANGES_NEXT_RELEASE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 5169f122e..10c4b4eb7 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Feature: do not propage TimeInstnat when explicitAttrs is empty array +- Fix: do not propage TimeInstant when explicitAttrs is empty array (#1606) - Fix: update device using previous device apikey to avoid error when apikey is updated (iota-json#833) - Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825, #1612) - Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) From ea2979dc4fad0afa4174a8151312331e373742a8 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 6 Jun 2024 12:56:52 +0200 Subject: [PATCH 81/83] Update doc/api.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- doc/api.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/api.md b/doc/api.md index f2b74c59b..c928422d4 100644 --- a/doc/api.md +++ b/doc/api.md @@ -464,8 +464,9 @@ element. By adding the field `explicitAttrs` with `true` value to device or conf measure elements that are not defined in the mappings of device or config group, persisting only the one defined in the mappings of the provision. If `explicitAttrs` is provided both at device and config group level, the device level takes precedence. Additionally `explicitAttrs` can be used to define which measures (identified by their attribute names, not -by their object_id) defined in JSON/JEXL array will be propagated to NGSI interface. When `explicitAttrs` is an array or -and JEXL expression resulting in to Array, if this array is empty then `TimeInstant` is not propaged to CB. +by their object_id) defined in JSON/JEXL array will be propagated to NGSI interface. + +Note that when `explicitAttrs` is an array or a JEXL expression resulting in to Array, if this array is empty then `TimeInstant` is not propaged to CB. The different possibilities are summarized below: From 1fed58b4efc20e81c49e9474b7572b7991a38514 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 11 Jun 2024 09:26:08 +0200 Subject: [PATCH 82/83] bump 4.5.0 --- CHANGES_NEXT_RELEASE | 5 ----- package.json | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 10c4b4eb7..e69de29bb 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,5 +0,0 @@ -- Fix: do not propage TimeInstant when explicitAttrs is empty array (#1606) -- Fix: update device using previous device apikey to avoid error when apikey is updated (iota-json#833) -- Fix: allow send multiple measures to CB in a batch (POST /v2/op/update) and sorted by TimeInstant when possible, instead of using multiples single request (iotagent-json#825, #1612) -- Fix: default express limit to 1Mb instead default 100Kb and allow change it throught a conf env var 'IOTA_EXPRESS_LIMIT' (iotagent-json#827) -- Fix: accept 201 status code from context broker with NGSI-LD interface when first measure is sent and device is created in it (#1617) diff --git a/package.json b/package.json index c9fc57981..dc507956b 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "iotagent-node-lib", "license": "AGPL-3.0-only", "description": "IoT Agent library to interface with NGSI Context Broker", - "version": "4.4.0-next", + "version": "4.5.0", "homepage": "https://github.com/telefonicaid/iotagent-node-lib", "keywords": [ "fiware", From 1c0faf30ed4135e56a94e73bb1523d7b962218f8 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Tue, 11 Jun 2024 10:32:36 +0200 Subject: [PATCH 83/83] step 4.5.0 -> 4.5.0-next --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index dc507956b..33b4ab0f4 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "iotagent-node-lib", "license": "AGPL-3.0-only", "description": "IoT Agent library to interface with NGSI Context Broker", - "version": "4.5.0", + "version": "4.5.0-next", "homepage": "https://github.com/telefonicaid/iotagent-node-lib", "keywords": [ "fiware",