Issue creating data notification messages with GXDLMSNotify.generateDataNotificationMessages

Forum Rules

Before commenting read Forum rules

Don't comment the topic if you have a new question.

You can create a new topic selecting correct category from Gurux Forum and then create a new topic selecting "New Topic" from the top left.

By Ryan Bressette , 26 August, 2022
Forums
Gurux.DLMS

My team has an executable written using the C++ GuruX library that creates data notification messages from metering data to be sent to our headend. I am attempting to translate this into Python to better mesh with our other libraries.

The executable takes the following input:

block_key = "00000000000000000000000000000001"
auth_key = "00000000000000000000000000000001"
wrapper_source = 1
wrapper_destination = 1
frame_counter = "139"
system_title = "931501000000FA03"

frame_data = '1E04630646947800010000008B00623C3C00000000FDAFFE83FE8E02 6304F5140000056400000000110A0173000000006306469400000AC800000000221402E7000000003CFD1800000078016B016B001F001F0000000078016B016B001F001F0000000082016B016B001F001F0000000082016B016B001F001F000000008C016B016B001F001F000000008C016B016B001F001F0000000096016B016B001F001F0000000096016B016B001F001F00000000A0016B016B001F001F00000000A0016B016B001F001F00000000AA016B016B001F001F00000000AA016B016B001F001F00000000B4016B016B001F001F00000000B4016B016B001F001F00000000BE016B016B001F001F00000000BE016B016B001F001F00000000C8016B016B001F001F00000000C8016B016B001F001F00000000D2016B016B001F001F00000000D2016B016B001F001F00000000DC016B016B001F001F00000000DC016B016B001F001F00000000E6016B016B001F001F00000000E6016B016B001F001F00'

The output is this which can be sent via UDP to the headend and what I expect the output of my Python code to be:

0001008B008B01A9DB08931501000000FA0382019C300000008BFB94A268BC2FE15620669772B5DF0570247E62C0AF98D5DD1BEE04F70E0E7EFF86EE9D46B15949CCE1BE8D6934787EEA9224E12F761F566C77E883B9B8C0F29F1C691321CEE8FFFCCA5B8A7E2E77189D4487810481BC1B30688A88845F132B4D036485A47873B66DB5935624C4A5F8E7062ACA2F667014FA915D3578B3C0048F770AC403299D54C1831C5393A76FF661D915F8A0C31EA844D1AF590463D3FE8B69FBCF201BC637E5D1050F37580BA07562819ABF9075E8DE0302E99FDE37CAFCDBF5531F150B0F7FB9CFBF9880F15CA040F10574B179FFB30B480F04F9FF76729462B6C424735180F103B277324CC243E794E29168543D10F6E13F10D03FC85C54385A970FB8290E78656079F831A3406EC8D2FF9FA7628EB363965512E1FAE7DA57067D37DB114858BAA96F7BF1F42C699464401E3EF3B26C46987B8149198FCCCA47BF6AAC1676AC35230F898795C1C66CBF896DF063359B9C74619D6D29D40184B2DFC4FA0A18E8A1840C379C459BBCD585B32BF80B1675DDF6E36E618FABB182D332E6A583F6741D51ED0AE8CB66C01F546A0A1E95

This is what I have based on the C++ code, but it generates an output that isn't even translatable at https://www.gurux.fi/GuruxDLMSTranslator:

import gurux_dlms
from gurux_dlms.GXDLMSNotify import GXDLMSNotify
from gurux_dlms.GXCiphering import GXCiphering
from gurux_dlms.enums import InterfaceType, Security, ServiceClass, Priority

block_key = "00000000000000000000000000000001"
auth_key = "00000000000000000000000000000001"
wrapper_source = 1
wrapper_destination = 1
frame_counter = "139"
system_title = "931501000000FA03"

block_key_bytes = gurux_dlms.GXByteBuffer(value=block_key)
auth_key_bytes = gurux_dlms.GXByteBuffer(value=auth_key)
system_title_bytes = gurux_dlms.GXByteBuffer(value=system_title)

frame_data = '1E04630646947800010000008B00623C3C00000000FDAFFE83FE8E02 6304F5140000056400000000110A0173000000006306469400000AC800000000221402E7000000003CFD1800000078016B016B001F001F0000000078016B016B001F001F0000000082016B016B001F001F0000000082016B016B001F001F000000008C016B016B001F001F000000008C016B016B001F001F0000000096016B016B001F001F0000000096016B016B001F001F00000000A0016B016B001F001F00000000A0016B016B001F001F00000000AA016B016B001F001F00000000AA016B016B001F001F00000000B4016B016B001F001F00000000B4016B016B001F001F00000000BE016B016B001F001F00000000BE016B016B001F001F00000000C8016B016B001F001F00000000C8016B016B001F001F00000000D2016B016B001F001F00000000D2016B016B001F001F00000000DC016B016B001F001F00000000DC016B016B001F001F00000000E6016B016B001F001F00000000E6016B016B001F001F00'

data_buffer = gurux_dlms.GXByteBuffer()
data_buffer.set(frame_data)

if __name__ == '__main__':
cipher = GXCiphering(title=bytearray.fromhex(system_title))
cipher.blockCipherKey = bytearray.fromhex(block_key)
cipher.authenticationKey = bytearray.fromhex(auth_key)
cipher.invocationCounter = frame_counter
cipher.setSecurity(Security.AUTHENTICATION_ENCRYPTION)

notify = GXDLMSNotify(useLogicalNameReferencing=True,
clientAddress=wrapper_source,
serverAddress=wrapper_destination,
interfaceType=InterfaceType.WRAPPER)
notify.setCipher(cipher)
notify.setInvokeID(0)
notify.setServiceClass(ServiceClass.UN_CONFIRMED)
notify.setPriority(Priority.NORMAL)
notify.setUseLogicalNameReferencing(True)

new_pdu = notify.generateDataNotificationMessages(None, data_buffer)

for elem in new_pdu:
print(elem.hex())

Erroneous output:
00010001000103050f00000001003145303436333036343639343738303030313030303030303842303036323343334330303030303030304644414646453833464538453032203633303446353134303030303035363430303030303030303131304130313733303030303030303036333036343639343030303030414338303030303030303032323134303245373030303030303030334346443138303030303030373830313642303136423030314630303146303030303030303037383031364230313642303031463030314630303030303030303832303136423031364230303146303031463030303030303030383230313642303136423030314630303146303030303030303038433031364230313642303031463030314630303030303030303843303136423031364230303146303031463030303030303030393630313642303136423030314630303146303030303030303039363031364230313642303031463030314630303030303030304130303136423031364230303146303031463030303030303030413030313642303136423030314630303146303030303030303041413031364230313642303031463030314630303030303030304141303136423031364230303146303031463030303030303030423430313642303136423030314630303146303030303030303042343031364230313642303031463030314630303030303030304245303136423031364230303146303031463030303030303030424530313642303136423030314630303146303030303030303043383031364230313642303031463030314630303030303030304338303136423031364230303146303031463030303030303030443230313642303136423030314630303146303030303030303044323031364230313642303031463030314630303030303030304443303136423031364230303146303031463030303030303030444330313642303136423030314630303146303030303030303045363031364230313642303031463030314630303030303030304536303136423031364230303146303031463030

Can someone assist me with this?

Thanks,

Ryan

Kurumi

3 years 9 months ago

Hi Ryan,

Online XML translator or python can't handle DiscoverReport at the moment. Try with GXDLMSDirector XML translator. It will handle your messages better.

I'll add DiscoverReport handling for ANSI C++ and python worklist. I can't say the exact date when it can be released at the moment.

BR,
Mikko

Ryan Bressette

3 years 9 months ago

Hi Mikko,

Forgive my ignorance, I do not have much experience with these libraries. I don't think this is an issue with the online translator or "DiscoverReport" (not sure what this is?)

It seems to be an issue with the Python version of the library or my input to its functions.

This is the working C++ code I am trying to translate:

std::string CompactFrameEncrypter::CompactFrameToDataNotification(const std::string& binaryCompactFrameBuffer)
{
unsigned char* bytes = new unsigned char[binaryCompactFrameBuffer.size()];
::memcpy(bytes, binaryCompactFrameBuffer.data(), binaryCompactFrameBuffer.size());
CGXDLMSVariant inner(bytes, binaryCompactFrameBuffer.size(), DLMS_DATA_TYPE_OCTET_STRING);
delete[] bytes;

CGXDLMSVariant outer;
outer.Arr.push_back(inner);
outer.vt = DLMS_DATA_TYPE_STRUCTURE;

CGXByteBuffer buf;
GXHelpers::SetData(buf, DLMS_DATA_TYPE_STRUCTURE, outer);

CGXByteBuffer blockKey;
blockKey.Set(m_blockKey.data(), m_blockKey.size());

CGXByteBuffer authKey;
authKey.Set(m_authKey.data(), m_authKey.size());

CGXByteBuffer systemTitle;
systemTitle.Set(m_systemTitle.data(), m_systemTitle.size());

auto cipher = std::make_unique<CGXCipher>(systemTitle);
cipher->SetBlockCipherKey(blockKey);
cipher->SetAuthenticationKey(authKey);
cipher->SetFrameCounter(m_frameCounter);
cipher->SetSecurity(DLMS_SECURITY_AUTHENTICATION_ENCRYPTION);

auto notify = std::make_unique<Notify>(cipher.get(), true, this->m_source, this->m_destination, DLMS_INTERFACE_TYPE_WRAPPER);
notify->SetInvokeID(0);
notify->SetServiceClass(DLMS_SERVICE_CLASS_UN_CONFIRMED);
notify->SetPriority(DLMS_PRIORITY_NORMAL);
notify->SetUseLogicalNameReferencing(true);

std::vector<CGXByteBuffer> reply;
notify->GenerateDataNotificationMessages(nullptr, buf, reply);

if (reply.size() != 1)
{
throw std::runtime_error("failed to generate data notification message");
}

auto msg = reply[0];
auto ptr = reinterpret_cast<char*>(msg.GetData());
return std::string(ptr, msg.GetSize());

Kurumi

3 years 9 months ago

Hi,
DiscoverReport is the payload that you want to send. PDU hex bytes converted to XML look like this:

<DiscoverReport>
<SystemTitle Value="63 06 46 94 78 00 01 00" />
<SystemTitle Value="00 00 8B 00 62 3C 3C 00" />
<SystemTitle Value="00 00 00 FD AF FE 83 FE" />
<SystemTitle Value="8E 02 63 04 F5 14 00 00" />
<AlarmDescriptor Value="100" />
</DiscoverReport>

The main problem is that GenerateDataNotificationMessages is used to send push messages and you want to generate DiscoverReport. Those are two totally different message types and the structure of the messages is different.

BR,
Mikko

Ryan Bressette

3 years 9 months ago

Hi,

frame_data = '1E04630646947800010000008B00623C3C00000000FDAFFE83FE8E02 6304F5140000056400000000110A0173000000006306469400000AC800000000221402E7000000003CFD1800000078016B016B001F001F0000000078016B016B001F001F0000000082016B016B001F001F0000000082016B016B001F001F000000008C016B016B001F001F000000008C016B016B001F001F0000000096016B016B001F001F0000000096016B016B001F001F00000000A0016B016B001F001F00000000A0016B016B001F001F00000000AA016B016B001F001F00000000AA016B016B001F001F00000000B4016B016B001F001F00000000B4016B016B001F001F00000000BE016B016B001F001F00000000BE016B016B001F001F00000000C8016B016B001F001F00000000C8016B016B001F001F00000000D2016B016B001F001F00000000D2016B016B001F001F00000000DC016B016B001F001F00000000DC016B016B001F001F00000000E6016B016B001F001F00000000E6016B016B001F001F00'

frame_data is data from our CF30 table that contains meter status, 2 register readings, and 24 interval readings. It is the payload that needs to be sent as a push message because that is how our meters push their metering data. That is how the C++ code works and it does exactly what I need.

Regards,
Ryan

Kurumi

3 years 9 months ago

Hi,

Is this the content of the compact data object or is it ciphered? The data types are missing from the data.

BR,
Mikko

Ryan Bressette

3 years 9 months ago

Hi,

This data is the hex for a table of metering data. It is the plaintext of the data my meters send to the headend in a data notification.

The exact content is irrelevant. My question is why the same functions that work for me in the C++ library (cipher, notify, and notify.generateDataNotificationMessages) does not give the correct output in Python?

Regards,
Ryan

Kurumi

3 years 9 months ago

Hi,

There was a typo that caused this. Get version 1.0.135 and you get the same result.

BR,
Mikko

Ryan Bressette

3 years 9 months ago

Hi,

I updated gurux_dlms to 1.0.135, but I am still not getting the result I'm expecting.

I'm at a loss at what to do to fix this.

Can you help me make the Python equivalent to this C++ code?

binaryCompactFrameBuffer = TableBuffer(result["pdu"].as<std::string>());
std::string m_blockKey = "00000000000000000000000000000001"
std::string m_authKey = "00000000000000000000000000000001"
std::string m_systemTitle = "931501000000FA03"
unsigned long m_frameCounter = 139
unsigned long m_source = 1
unsigned long m_destination = 1

std::string CompactFrameEncrypter::CompactFrameToDataNotification(
const std::string& binaryCompactFrameBuffer,
std::string m_blockKey,
std::string m_authKey,
std::string m_systemTitle,
unsigned long m_frameCounter,
unsigned long m_source,
unsigned long m_destination)
{

unsigned char* bytes = new unsigned char[binaryCompactFrameBuffer.size()];
::memcpy(bytes, binaryCompactFrameBuffer.data(), binaryCompactFrameBuffer.size());
CGXDLMSVariant inner(bytes, binaryCompactFrameBuffer.size(), DLMS_DATA_TYPE_OCTET_STRING);
delete[] bytes;

CGXDLMSVariant outer;
outer.Arr.push_back(inner);
outer.vt = DLMS_DATA_TYPE_STRUCTURE;
CGXByteBuffer buf;
GXHelpers::SetData(buf, DLMS_DATA_TYPE_STRUCTURE, outer);

CGXByteBuffer blockKey;
blockKey.Set(m_blockKey.data(), m_blockKey.size());

CGXByteBuffer authKey;
authKey.Set(m_authKey.data(), m_authKey.size());

CGXByteBuffer systemTitle;
systemTitle.Set(m_systemTitle.data(), m_systemTitle.size());

auto cipher = std::make_unique<CGXCipher>(systemTitle);
cipher->SetBlockCipherKey(blockKey);
cipher->SetAuthenticationKey(authKey);
cipher->SetFrameCounter(m_frameCounter);
cipher->SetSecurity(DLMS_SECURITY_AUTHENTICATION_ENCRYPTION);

auto notify = std::make_unique<Notify>(cipher.get(), true, m_source, m_destination, DLMS_INTERFACE_TYPE_WRAPPER);
notify->SetInvokeID(0);
notify->SetServiceClass(DLMS_SERVICE_CLASS_UN_CONFIRMED);
notify->SetPriority(DLMS_PRIORITY_NORMAL);
notify->SetUseLogicalNameReferencing(true);

std::vector<CGXByteBuffer> reply;
notify->GenerateDataNotificationMessages(nullptr, buf, reply);

if (reply.size() != 1)
{
throw std::runtime_error("failed to generate data notification message");
}
auto msg = reply[0];
auto ptr = reinterpret_cast<char*>(msg.GetData());
return std::string(ptr, msg.GetSize());

Regards,
Ryan

Kurumi

3 years 9 months ago

Hi,

Update to version 1.0.136. You need to add data type that is missing at the moment and for that reason DLMS Translator doesn't work with this. The data type is missing. This is working code:

system_title = GXByteBuffer.hexToBytes("931501000000FA03")
block_key = GXByteBuffer.hexToBytes("00000000000000000000000000000001")
auth_key = GXByteBuffer.hexToBytes("00000000000000000000000000000001")
notify = GXDLMSSecureNotify(True, 1, 1, InterfaceType.WRAPPER)
ciphering = notify.getCiphering()
ciphering.systemTitle = system_title
ciphering.blockCipherKey = block_key
ciphering.authenticationKey = auth_key
ciphering.invocationCounter = 139
ciphering.security = Security.AUTHENTICATION_ENCRYPTION
frame_data = GXByteBuffer.hexToBytes('1E04630646947800010000008B00623C3C00000000FDAFFE83FE8E02 6304F5140000056400000000110A0173000000006306469400000AC800000000221402E7000000003CFD1800000078016B016B001F001F0000000078016B016B001F001F0000000082016B016B001F001F0000000082016B016B001F001F000000008C016B016B001F001F000000008C016B016B001F001F0000000096016B016B001F001F0000000096016B016B001F001F00000000A0016B016B001F001F00000000A0016B016B001F001F00000000AA016B016B001F001F00000000AA016B016B001F001F00000000B4016B016B001F001F00000000B4016B016B001F001F00000000BE016B016B001F001F00000000BE016B016B001F001F00000000C8016B016B001F001F00000000C8016B016B001F001F00000000D2016B016B001F001F00000000D2016B016B001F001F00000000DC016B016B001F001F00000000DC016B016B001F001F00000000E6016B016B001F001F00000000E6016B016B001F001F00')
data_buffer = GXByteBuffer()

#Add data type.
data_buffer.setUInt8(DataType.STRUCTURE)
#Add count
data_buffer.setUInt8(1)
#Add data as octet-string.
data_buffer.setUInt8(DataType.OCTET_STRING)
#Data length.
_GXCommon.setObjectCount(len(frame_data), data_buffer)
data_buffer.set(frame_data)
new_pdu = notify.generateDataNotificationMessages(None, data_buffer)
for elem in new_pdu:
print(elem.hex())

BR,
Mikko