Skip to content

Commit

Permalink
Supporting LF as dataset end marker
Browse files Browse the repository at this point in the history
  • Loading branch information
@lains
lains committed Apr 10, 2024
1 parent 0a3349c commit 5e52796
Show file tree
Hide file tree
Showing 6 changed files with 174 additions and 23 deletions.
3 changes: 2 additions & 1 deletion include/TIC/DatasetExtractor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ class DatasetExtractor {
static constexpr uint8_t LF = 0x0a;
static constexpr uint8_t CR = 0x0d;
static constexpr uint8_t START_MARKER = LF; /*!< Dataset start marker (line feed) */
static constexpr uint8_t END_MARKER = CR; /*!< Dataset end marker (carriage return) */
static constexpr uint8_t END_MARKER_TIC_1 = CR; /*!< 1st possible dataset end marker for TIC (carriage return) */
static constexpr uint8_t END_MARKER_TIC_2 = LF; /*!< 2nd possible dataset end marker for TIC (line feed) */
static constexpr unsigned int MAX_DATASET_SIZE = 128; /*!< Max size for a dataset storage (in bytes) */

/* Methods */
Expand Down
24 changes: 22 additions & 2 deletions src/TIC/DatasetExtractor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ unsigned int TIC::DatasetExtractor::pushBytes(const uint8_t* buffer, unsigned in
*/
if (len == 0)
return 0;
//std::cout << "de got " << len << " bytes starting with: " << std::hex << static_cast<unsigned int>(buffer[0]) << " while " << std::string(this->sync?"inside":"outside") << " of a dataset\n";
unsigned int usedBytes = 0;
if (!this->sync) { /* We don't record bytes, we'll just look for a start of dataset */
uint8_t* firstStartOfDataset = (uint8_t*)(memchr(buffer, TIC::DatasetExtractor::START_MARKER, len));
Expand All @@ -41,7 +42,23 @@ unsigned int TIC::DatasetExtractor::pushBytes(const uint8_t* buffer, unsigned in
}
else {
/* We are inside a TIC dataset, search for the end of dataset marker */
uint8_t* endOfDataset = (uint8_t*)(memchr(buffer, TIC::DatasetExtractor::END_MARKER, len)); /* Search for end of dataset */
//FIXME: historical TIC uses LF, standard TIC uses CR
//If we allow LD below, and we have transmission errors, we may become out of sync if we catch a wrong extraneous LF... we will be out of sync and get only empty datasets because we swap start and end markers
//We should have a way to recover from this, for example by detecting 0 size datasets and thus understand we need to invert sync state
uint8_t* probeEndOfDataset1 = nullptr; /* Attempt to detect the end of a dataset formatted with standard TIC */
uint8_t* probeEndOfDataset2 = nullptr; /* Attempt to detect the end of a dataset formatted with historical TIC */
uint8_t* endOfDataset = nullptr;
probeEndOfDataset1 = (uint8_t*)(memchr(buffer, TIC::DatasetExtractor::END_MARKER_TIC_1, len));
if (probeEndOfDataset1) {
endOfDataset = probeEndOfDataset1;
}
else {
probeEndOfDataset2 = (uint8_t*)(memchr(buffer, TIC::DatasetExtractor::END_MARKER_TIC_2, len)); /* Search for end of dataset formatted with historical TIC */
if (probeEndOfDataset2) {
endOfDataset = probeEndOfDataset2;
}
}

if (endOfDataset) { /* We have an end of dataset marker in the buffer, we can extract the full dataset */
unsigned int leadingBytesInPreviousDataset = endOfDataset - buffer;
usedBytes = this->processIncomingDatasetBytes(buffer, leadingBytesInPreviousDataset, true); /* Copy the buffer up to (but exclusing the end of dataset marker), the dataset is complete */
Expand Down Expand Up @@ -76,7 +93,10 @@ unsigned int TIC::DatasetExtractor::processIncomingDatasetBytes(const uint8_t* b
}

void TIC::DatasetExtractor::processCurrentDataset() {
this->onDatasetExtracted(this->currentDataset, this->nextWriteInCurrentDataset, this->onDatasetExtractedContext);
//std::vector<uint8_t> datasetContent(this->currentDataset, this->currentDataset+this->nextWriteInCurrentDataset);
//std::cout << "New dataset extracted: " << vectorToHexString(datasetContent) << " (as string: \"" << std::string(datasetContent.begin(), datasetContent.end()) << "\")\n";
if (this->onDatasetExtracted)
this->onDatasetExtracted(this->currentDataset, this->nextWriteInCurrentDataset, this->onDatasetExtractedContext);
}

bool TIC::DatasetExtractor::isInSync() const {
Expand Down
28 changes: 21 additions & 7 deletions src/TIC/Unframer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,18 +46,31 @@ unsigned int TIC::Unframer::pushBytes(const uint8_t* buffer, unsigned int len) {
else {
/* We are inside a TIC frame, search for the end of frame (ETX) marker */
uint8_t* etx = (uint8_t*)(memchr(buffer, TIC::Unframer::END_MARKER, len)); /* Search for end of frame */
if (etx) { /* We have an ETX in the buffer, we can extract the full frame */
unsigned int leadingBytesInPreviousFrame = etx - buffer;
usedBytes = this->processIncomingFrameBytes(buffer, leadingBytesInPreviousFrame); /* Copy the buffer up to (but exclusing the ETX marker) */
uint8_t* stx = nullptr;
if (!etx) { /* Historical TIC may not contain any ETX, but a STX would also signify that the current frame is over */
stx = (uint8_t*)(memchr(buffer, TIC::Unframer::START_MARKER, len)); /* Search for a start if not end of frame is found */
}
if (etx || stx) { /* We have detected the end of the current frame in the buffer, we can extract the full frame */
//std::cout << std::string("Got ") << std::string(etx?"end marker":"unexpected start") << "\n";
unsigned int leadingBytesInPreviousFrame;
if (etx) {
leadingBytesInPreviousFrame = etx - buffer;
}
else if (stx) {
leadingBytesInPreviousFrame = stx - buffer;
}
usedBytes = this->processIncomingFrameBytes(buffer, leadingBytesInPreviousFrame); /* Copy the buffer up to (but exclusing the end of frame marker) */
this->processCurrentFrame(); /* The frame is complete */
leadingBytesInPreviousFrame++; /* Skip the ETX marker */
usedBytes++;
if (etx) {
leadingBytesInPreviousFrame++; /* Skip the ETX marker */
usedBytes++;
}
this->sync = false; /* Consider we are outside of a frame now */
if (leadingBytesInPreviousFrame < len) { /* We have at least one byte after the frame ETX */
usedBytes += this->pushBytes(buffer + leadingBytesInPreviousFrame, len - leadingBytesInPreviousFrame); /* Process the trailing bytes (probably the next frame, starting with STX) */
}
}
else { /* No ETX, copy the whole chunk */
else { /* No end of frame was found, copy the whole chunk */
/* Incoming bytes */
usedBytes = this->processIncomingFrameBytes(buffer, len); /* Process these bytes as valid */
}
Expand Down Expand Up @@ -90,8 +103,9 @@ void TIC::Unframer::processCurrentFrame() {
this->onNewFrameBytes(this->currentFrame, this->nextWriteInCurrentFrame, this->parserFuncContext);
this->nextWriteInCurrentFrame = 0; /* Wipe any data in the current frame, start over */
#endif
if (this->onFrameComplete != nullptr)
if (this->onFrameComplete != nullptr) {
this->onFrameComplete(this->parserFuncContext);
}
}

bool TIC::Unframer::isInSync() const {
Expand Down
139 changes: 127 additions & 12 deletions test/src/TicDatasetExtractor_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <string>
#include <iterator>
#include <cstring>
#include <iomanip>

#include "Tools.h"
#include "TIC/DatasetExtractor.h"
Expand Down Expand Up @@ -60,6 +61,11 @@ static void datasetExtractorUnwrapForwardFrameBytes(const uint8_t* buf, unsigned
if (context == NULL)
return; /* Failsafe, discard if no context */
TIC::DatasetExtractor* de = static_cast<TIC::DatasetExtractor*>(context);
// std::cout << "Got " << cnt << " bytes: ";
// for (unsigned int i = 0; i < cnt; i++) {
// std::cout << std::hex << std::setfill('0') << std::setw(2) << static_cast<unsigned int>(buf[i]) << " ";
// }
// std::cout << "\n";
de->pushBytes(buf, cnt);
}

Expand All @@ -72,6 +78,7 @@ static void datasetExtractorUnWrapFrameFinished(void *context) {
if (context == NULL)
return; /* Failsafe, discard if no context */
TIC::DatasetExtractor* de = static_cast<TIC::DatasetExtractor*>(context);
// std::cout << "Frame finished\n";
/* We have finished parsing a frame, if there is an open dataset, we should discard it and start over at the following frame */
de->reset();
}
Expand All @@ -89,9 +96,9 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_dataset_10byte
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes) {
TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type1) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_1;
uint8_t buffer[] = { 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
Expand All @@ -106,9 +113,45 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes) {
TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type2) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_2;
uint8_t buffer[] = { 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
de.pushBytes(&start_marker, 1);
de.pushBytes(buffer, sizeof(buffer));
de.pushBytes(&end_marker, 1);
if (stub.decodedDatasetList.size() != 1) {
FAILF("Wrong dataset count: %zu\nDatasets received:\n%s", stub.decodedDatasetList.size(), stub.toString().c_str());
}
if (stub.decodedDatasetList[0] != std::vector<uint8_t>({0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39})) {
FAILF("Wrong dataset decoded: %s", vectorToHexString(stub.decodedDatasetList[0]).c_str());
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type1) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_1;
uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
de.pushBytes(&start_marker, 1);
for (unsigned int pos = 0; pos < sizeof(buffer); pos++) {
de.pushBytes(buffer + pos, 1);
}
de.pushBytes(&end_marker, 1);
if (stub.decodedDatasetList.size() != 1) {
FAILF("Wrong dataset count: %zu\nDatasets received:\n%s", stub.decodedDatasetList.size(), stub.toString().c_str());
}
if (stub.decodedDatasetList[0] != std::vector<uint8_t>({0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39})) {
FAILF("Wrong dataset decoded: %s", vectorToHexString(stub.decodedDatasetList[0]).c_str());
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type2) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_2;
uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
Expand All @@ -131,11 +174,11 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_
buffer[0] = TIC::DatasetExtractor::START_MARKER;
for (unsigned int pos = 1; pos < sizeof(buffer) - 1 ; pos++) {
buffer[pos] = (uint8_t)(pos & 0xff);
if (buffer[pos] == TIC::DatasetExtractor::START_MARKER || buffer[pos] == TIC::DatasetExtractor::END_MARKER || buffer[pos] == TIC::Unframer::START_MARKER || buffer[pos] == TIC::Unframer::END_MARKER) {
if (buffer[pos] == TIC::DatasetExtractor::START_MARKER || buffer[pos] == TIC::DatasetExtractor::END_MARKER_TIC_1 || buffer[pos] == TIC::Unframer::START_MARKER || buffer[pos] == TIC::Unframer::END_MARKER) {
buffer[pos] = 0x00; /* Remove any frame of dataset delimiters */
}
}
buffer[sizeof(buffer) - 1] = TIC::DatasetExtractor::END_MARKER;
buffer[sizeof(buffer) - 1] = TIC::DatasetExtractor::END_MARKER_TIC_1;
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
de.pushBytes(buffer, sizeof(buffer) / 2);
Expand All @@ -148,9 +191,28 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves) {
TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type1) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_1;
uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
de.pushBytes(&start_marker, 1);
for (uint8_t pos = 0; pos < sizeof(buffer); pos++) {
de.pushBytes(buffer + pos, 1);
}
de.pushBytes(&end_marker, 1);
if (stub.decodedDatasetList.size() != 1) {
FAILF("Wrong dataset count: %zu\nDatasets received:\n%s", stub.decodedDatasetList.size(), stub.toString().c_str());
}
if (stub.decodedDatasetList[0] != std::vector<uint8_t>({0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39})) {
FAILF("Wrong dataset decoded: %s", vectorToHexString(stub.decodedDatasetList[0]).c_str());
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type2) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_2;
uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
Expand Down Expand Up @@ -212,7 +274,56 @@ TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_historical_TIC)
expectedTotalDatasetCount += 6; /* 6 more trailing dataset within an unterminated frame */
#endif

if (stub.decodedDatasetList.size() != expectedTotalDatasetCount) {
if (stub.decodedDatasetList.size() != expectedTotalDatasetCount) {
FAILF("When using chunk size %u: Wrong dataset count: %zu, expected %zu\nDatasets received:\n%s", chunkSize, stub.decodedDatasetList.size(), expectedTotalDatasetCount, stub.toString().c_str());
}
char firstDatasetAsCString[] = "ADCO 056234673197 L";
std::vector<uint8_t> expectedFirstDatasetInFrame(firstDatasetAsCString, firstDatasetAsCString+strlen(firstDatasetAsCString));
if (stub.decodedDatasetList[0] != expectedFirstDatasetInFrame) {
FAILF("Unexpected first dataset in first frame:\nGot: %s\nExpected: %s\n", vectorToHexString(stub.decodedDatasetList[0]).c_str(), vectorToHexString(expectedFirstDatasetInFrame).c_str());
}
char lastDatasetAsCString[] = "PPOT 00 #";
std::vector<uint8_t> expectedLastDatasetInFrame(lastDatasetAsCString, lastDatasetAsCString+strlen(lastDatasetAsCString));
if (stub.decodedDatasetList[nbExpectedDatasetPerFrame-1] != expectedLastDatasetInFrame) {
FAILF("Unexpected last dataset in first frame:\nGot: %s\nExpected: %s\n", vectorToHexString(stub.decodedDatasetList[nbExpectedDatasetPerFrame-1]).c_str(), vectorToHexString(expectedLastDatasetInFrame).c_str());
}
for (std::size_t datasetIndex = 0; datasetIndex < stub.decodedDatasetList.size(); datasetIndex++) {
std::size_t receivedDatasetSize = stub.decodedDatasetList[datasetIndex].size();
std::size_t expectedDatasetSize = datasetExpectedSizes[datasetIndex % nbExpectedDatasetPerFrame];
if (receivedDatasetSize != expectedDatasetSize) {
FAILF("When using chunk size %u: Wrong dataset decoded at index %zu in frame. Expected %zu bytes, got %zu bytes. Dataset content: %s", chunkSize, datasetIndex, expectedDatasetSize, receivedDatasetSize, vectorToHexString(stub.decodedDatasetList[datasetIndex]).c_str());
}
}
}
}

TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_historical_TIC_2) {
std::vector<uint8_t> ticData = readVectorFromDisk("./samples/continuous_linky_3P_historical_TIC_2024_sample.bin");

for (unsigned int chunkSize = 1; chunkSize <= TIC::DatasetExtractor::MAX_DATASET_SIZE; chunkSize++) {
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
TIC::Unframer tu(datasetExtractorUnwrapForwardFrameBytes, datasetExtractorUnWrapFrameFinished, &de);

TicUnframer_test_file_sent_by_chunks(ticData, chunkSize, tu);

/**
* @brief Sizes (in bytes) of the successive dataset in each repeated TIC frame
*/
std::size_t datasetExpectedSizes[] = { 19, /* ADCO label */
14, 11, 16, 11,
12, 12, 12, /* Three times IINST? labels (on each phase) */
11, 11, 11, /* Three times IMAX? labels (on each phase) */
12, 12, 9, 17, 9
};
unsigned int nbExpectedDatasetPerFrame = sizeof(datasetExpectedSizes)/sizeof(datasetExpectedSizes[0]);

std::size_t expectedTotalDatasetCount = 4 * nbExpectedDatasetPerFrame; /* 4 frames, each containing the above datasets */
#ifdef __TIC_UNFRAMER_FORWARD_FRAME_BYTES_ON_THE_FLY__
expectedTotalDatasetCount += 7; /* 7 more trailing dataset within an unterminated frame */
#endif

if (stub.decodedDatasetList.size() != expectedTotalDatasetCount) {
FAILF("When using chunk size %u: Wrong dataset count: %zu, expected %zu\nDatasets received:\n%s", chunkSize, stub.decodedDatasetList.size(), expectedTotalDatasetCount, stub.toString().c_str());
}
char firstDatasetAsCString[] = "ADCO 056234673197 L";
Expand Down Expand Up @@ -288,11 +399,15 @@ TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_standard_TIC) {
#ifndef USE_CPPUTEST
void runTicDatasetExtractorAllUnitTests() {
TicDatasetExtractor_test_one_pure_dataset_10bytes();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type1();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type2();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type1();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type2();
TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_max_buffer();
TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves();
TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type1();
TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type2();
Chunked_sample_unframe_dsextract_historical_TIC();
Chunked_sample_unframe_dsextract_historical_TIC_2();
Chunked_sample_unframe_dsextract_standard_TIC();
}
#endif // USE_CPPUTEST
2 changes: 1 addition & 1 deletion test/src/TicDatasetView_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ TEST(TicDatasetView_tests, TicDatasetView_extra_leading_start_marker) {

TEST(TicDatasetView_tests, TicDatasetView_extra_trailing_end_marker) {
char dataset[] = { "ADCO 012345678901 E*"};
dataset[sizeof(dataset)-1-1] = TIC::DatasetExtractor::END_MARKER; /* Replace the * with our end marker (-1 to get inside the buffer, -1 again to move before the terminating '\0') */
dataset[sizeof(dataset)-1-1] = TIC::DatasetExtractor::END_MARKER_TIC_1; /* Replace the * with our end marker (-1 to get inside the buffer, -1 again to move before the terminating '\0') */

const uint8_t* datasetBuf = reinterpret_cast<const unsigned char*>(dataset);

Expand Down
1 change: 1 addition & 0 deletions test/src/Tools.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <vector>
#include <string>
#include <fstream>
#include "stdint.h"

std::string vectorToHexString(const std::vector<uint8_t> &vec);

Expand Down

0 comments on commit 5e52796

Please sign in to comment.