Skip to content

Commit

Permalink
Supporting LF as dataset end marker
Browse files Browse the repository at this point in the history
  • Loading branch information
@lains
lains committed Apr 10, 2024
1 parent a9c795b commit 6cede60
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 16 deletions.
3 changes: 2 additions & 1 deletion include/TIC/DatasetExtractor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ class DatasetExtractor {
static constexpr uint8_t LF = 0x0a;
static constexpr uint8_t CR = 0x0d;
static constexpr uint8_t START_MARKER = LF; /*!< Dataset start marker (line feed) */
static constexpr uint8_t END_MARKER = CR; /*!< Dataset end marker (carriage return) */
static constexpr uint8_t END_MARKER_TIC_1 = CR; /*!< 1st possible dataset end marker for TIC (carriage return) */
static constexpr uint8_t END_MARKER_TIC_2 = LF; /*!< 2nd possible dataset end marker for TIC (line feed) */
static constexpr unsigned int MAX_DATASET_SIZE = 128; /*!< Max size for a dataset storage (in bytes) */

/* Methods */
Expand Down
15 changes: 14 additions & 1 deletion src/TIC/DatasetExtractor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,20 @@ unsigned int TIC::DatasetExtractor::pushBytes(const uint8_t* buffer, unsigned in
//FIXME: historical TIC uses LF, standard TIC uses CR
//If we allow LD below, and we have transmission errors, we may become out of sync if we catch a wrong extraneous LF... we will be out of sync and get only empty datasets because we swap start and end markers
//We should have a way to recover from this, for example by detecting 0 size datasets and thus understand we need to invert sync state
uint8_t* endOfDataset = (uint8_t*)(memchr(buffer, TIC::DatasetExtractor::LF, len)); /* Search for end of dataset */
uint8_t* probeEndOfDataset1 = nullptr; /* Attempt to detect the end of a dataset formatted with standard TIC */
uint8_t* probeEndOfDataset2 = nullptr; /* Attempt to detect the end of a dataset formatted with historical TIC */
uint8_t* endOfDataset = nullptr;
probeEndOfDataset1 = (uint8_t*)(memchr(buffer, TIC::DatasetExtractor::END_MARKER_TIC_1, len));
if (probeEndOfDataset1) {
endOfDataset = probeEndOfDataset1;
}
else {
probeEndOfDataset2 = (uint8_t*)(memchr(buffer, TIC::DatasetExtractor::END_MARKER_TIC_2, len)); /* Search for end of dataset formatted with historical TIC */
if (probeEndOfDataset2) {
endOfDataset = probeEndOfDataset2;
}
}

if (endOfDataset) { /* We have an end of dataset marker in the buffer, we can extract the full dataset */
unsigned int leadingBytesInPreviousDataset = endOfDataset - buffer;
usedBytes = this->processIncomingDatasetBytes(buffer, leadingBytesInPreviousDataset, true); /* Copy the buffer up to (but exclusing the end of dataset marker), the dataset is complete */
Expand Down
134 changes: 121 additions & 13 deletions test/src/TicDatasetExtractor_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,9 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_dataset_10byte
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes) {
TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type1) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_1;
uint8_t buffer[] = { 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
Expand All @@ -113,9 +113,45 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes) {
TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type2) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_2;
uint8_t buffer[] = { 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
de.pushBytes(&start_marker, 1);
de.pushBytes(buffer, sizeof(buffer));
de.pushBytes(&end_marker, 1);
if (stub.decodedDatasetList.size() != 1) {
FAILF("Wrong dataset count: %zu\nDatasets received:\n%s", stub.decodedDatasetList.size(), stub.toString().c_str());
}
if (stub.decodedDatasetList[0] != std::vector<uint8_t>({0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39})) {
FAILF("Wrong dataset decoded: %s", vectorToHexString(stub.decodedDatasetList[0]).c_str());
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type1) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_1;
uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
de.pushBytes(&start_marker, 1);
for (unsigned int pos = 0; pos < sizeof(buffer); pos++) {
de.pushBytes(buffer + pos, 1);
}
de.pushBytes(&end_marker, 1);
if (stub.decodedDatasetList.size() != 1) {
FAILF("Wrong dataset count: %zu\nDatasets received:\n%s", stub.decodedDatasetList.size(), stub.toString().c_str());
}
if (stub.decodedDatasetList[0] != std::vector<uint8_t>({0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39})) {
FAILF("Wrong dataset decoded: %s", vectorToHexString(stub.decodedDatasetList[0]).c_str());
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type2) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_2;
uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
Expand All @@ -138,11 +174,11 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_
buffer[0] = TIC::DatasetExtractor::START_MARKER;
for (unsigned int pos = 1; pos < sizeof(buffer) - 1 ; pos++) {
buffer[pos] = (uint8_t)(pos & 0xff);
if (buffer[pos] == TIC::DatasetExtractor::START_MARKER || buffer[pos] == TIC::DatasetExtractor::END_MARKER || buffer[pos] == TIC::Unframer::START_MARKER || buffer[pos] == TIC::Unframer::END_MARKER) {
if (buffer[pos] == TIC::DatasetExtractor::START_MARKER || buffer[pos] == TIC::DatasetExtractor::END_MARKER_TIC_1 || buffer[pos] == TIC::Unframer::START_MARKER || buffer[pos] == TIC::Unframer::END_MARKER) {
buffer[pos] = 0x00; /* Remove any frame of dataset delimiters */
}
}
buffer[sizeof(buffer) - 1] = TIC::DatasetExtractor::END_MARKER;
buffer[sizeof(buffer) - 1] = TIC::DatasetExtractor::END_MARKER_TIC_1;
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
de.pushBytes(buffer, sizeof(buffer) / 2);
Expand All @@ -155,9 +191,28 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves) {
TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type1) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_1;
uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
de.pushBytes(&start_marker, 1);
for (uint8_t pos = 0; pos < sizeof(buffer); pos++) {
de.pushBytes(buffer + pos, 1);
}
de.pushBytes(&end_marker, 1);
if (stub.decodedDatasetList.size() != 1) {
FAILF("Wrong dataset count: %zu\nDatasets received:\n%s", stub.decodedDatasetList.size(), stub.toString().c_str());
}
if (stub.decodedDatasetList[0] != std::vector<uint8_t>({0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39})) {
FAILF("Wrong dataset decoded: %s", vectorToHexString(stub.decodedDatasetList[0]).c_str());
}
}

TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type2) {
uint8_t start_marker = TIC::DatasetExtractor::START_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER;
uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_2;
uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 };
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
Expand Down Expand Up @@ -194,7 +249,7 @@ static void TicUnframer_test_file_sent_by_chunks(const std::vector<uint8_t>& tic
}

TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_historical_TIC) {
std::vector<uint8_t> ticData = readVectorFromDisk("./samples/continuous_linky_3P_historical_TIC_2024_sample.bin");
std::vector<uint8_t> ticData = readVectorFromDisk("./samples/continuous_linky_3P_historical_TIC_sample.bin");

for (unsigned int chunkSize = 1; chunkSize <= TIC::DatasetExtractor::MAX_DATASET_SIZE; chunkSize++) {
DatasetDecoderStub stub;
Expand All @@ -219,7 +274,56 @@ TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_historical_TIC)
expectedTotalDatasetCount += 6; /* 6 more trailing dataset within an unterminated frame */
#endif

if (stub.decodedDatasetList.size() != expectedTotalDatasetCount) {
if (stub.decodedDatasetList.size() != expectedTotalDatasetCount) {
FAILF("When using chunk size %u: Wrong dataset count: %zu, expected %zu\nDatasets received:\n%s", chunkSize, stub.decodedDatasetList.size(), expectedTotalDatasetCount, stub.toString().c_str());
}
char firstDatasetAsCString[] = "ADCO 056234673197 L";
std::vector<uint8_t> expectedFirstDatasetInFrame(firstDatasetAsCString, firstDatasetAsCString+strlen(firstDatasetAsCString));
if (stub.decodedDatasetList[0] != expectedFirstDatasetInFrame) {
FAILF("Unexpected first dataset in first frame:\nGot: %s\nExpected: %s\n", vectorToHexString(stub.decodedDatasetList[0]).c_str(), vectorToHexString(expectedFirstDatasetInFrame).c_str());
}
char lastDatasetAsCString[] = "PPOT 00 #";
std::vector<uint8_t> expectedLastDatasetInFrame(lastDatasetAsCString, lastDatasetAsCString+strlen(lastDatasetAsCString));
if (stub.decodedDatasetList[nbExpectedDatasetPerFrame-1] != expectedLastDatasetInFrame) {
FAILF("Unexpected last dataset in first frame:\nGot: %s\nExpected: %s\n", vectorToHexString(stub.decodedDatasetList[nbExpectedDatasetPerFrame-1]).c_str(), vectorToHexString(expectedLastDatasetInFrame).c_str());
}
for (std::size_t datasetIndex = 0; datasetIndex < stub.decodedDatasetList.size(); datasetIndex++) {
std::size_t receivedDatasetSize = stub.decodedDatasetList[datasetIndex].size();
std::size_t expectedDatasetSize = datasetExpectedSizes[datasetIndex % nbExpectedDatasetPerFrame];
if (receivedDatasetSize != expectedDatasetSize) {
FAILF("When using chunk size %u: Wrong dataset decoded at index %zu in frame. Expected %zu bytes, got %zu bytes. Dataset content: %s", chunkSize, datasetIndex, expectedDatasetSize, receivedDatasetSize, vectorToHexString(stub.decodedDatasetList[datasetIndex]).c_str());
}
}
}
}

TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_historical_TIC_2) {
std::vector<uint8_t> ticData = readVectorFromDisk("./samples/continuous_linky_3P_historical_TIC_2024_sample.bin");

for (unsigned int chunkSize = 1; chunkSize <= TIC::DatasetExtractor::MAX_DATASET_SIZE; chunkSize++) {
DatasetDecoderStub stub;
TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub);
TIC::Unframer tu(datasetExtractorUnwrapForwardFrameBytes, datasetExtractorUnWrapFrameFinished, &de);

TicUnframer_test_file_sent_by_chunks(ticData, chunkSize, tu);

/**
* @brief Sizes (in bytes) of the successive dataset in each repeated TIC frame
*/
std::size_t datasetExpectedSizes[] = { 19, /* ADCO label */
14, 11, 16, 11,
12, 12, 12, /* Three times IINST? labels (on each phase) */
11, 11, 11, /* Three times IMAX? labels (on each phase) */
12, 12, 9, 17, 9
};
unsigned int nbExpectedDatasetPerFrame = sizeof(datasetExpectedSizes)/sizeof(datasetExpectedSizes[0]);

std::size_t expectedTotalDatasetCount = 4 * nbExpectedDatasetPerFrame; /* 4 frames, each containing the above datasets */
#ifdef __TIC_UNFRAMER_FORWARD_FRAME_BYTES_ON_THE_FLY__
expectedTotalDatasetCount += 7; /* 7 more trailing dataset within an unterminated frame */
#endif

if (stub.decodedDatasetList.size() != expectedTotalDatasetCount) {
FAILF("When using chunk size %u: Wrong dataset count: %zu, expected %zu\nDatasets received:\n%s", chunkSize, stub.decodedDatasetList.size(), expectedTotalDatasetCount, stub.toString().c_str());
}
char firstDatasetAsCString[] = "ADCO 056234673197 L";
Expand Down Expand Up @@ -295,11 +399,15 @@ TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_standard_TIC) {
#ifndef USE_CPPUTEST
void runTicDatasetExtractorAllUnitTests() {
TicDatasetExtractor_test_one_pure_dataset_10bytes();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type1();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type2();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type1();
TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type2();
TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_max_buffer();
TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves();
TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type1();
TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type2();
Chunked_sample_unframe_dsextract_historical_TIC();
Chunked_sample_unframe_dsextract_historical_TIC_2();
Chunked_sample_unframe_dsextract_standard_TIC();
}
#endif // USE_CPPUTEST
2 changes: 1 addition & 1 deletion test/src/TicDatasetView_tests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ TEST(TicDatasetView_tests, TicDatasetView_extra_leading_start_marker) {

TEST(TicDatasetView_tests, TicDatasetView_extra_trailing_end_marker) {
char dataset[] = { "ADCO 012345678901 E*"};
dataset[sizeof(dataset)-1-1] = TIC::DatasetExtractor::END_MARKER; /* Replace the * with our end marker (-1 to get inside the buffer, -1 again to move before the terminating '\0') */
dataset[sizeof(dataset)-1-1] = TIC::DatasetExtractor::END_MARKER_TIC_1; /* Replace the * with our end marker (-1 to get inside the buffer, -1 again to move before the terminating '\0') */

const uint8_t* datasetBuf = reinterpret_cast<const unsigned char*>(dataset);

Expand Down

0 comments on commit 6cede60

Please sign in to comment.