From 6cede602b71039dfe2df3307937568e5a9bc2b1d Mon Sep 17 00:00:00 2001 From: lains <383502+lains@users.noreply.github.com> Date: Wed, 10 Apr 2024 09:39:20 +0200 Subject: [PATCH] Supporting LF as dataset end marker --- include/TIC/DatasetExtractor.h | 3 +- src/TIC/DatasetExtractor.cpp | 15 ++- test/src/TicDatasetExtractor_tests.cpp | 134 ++++++++++++++++++++++--- test/src/TicDatasetView_tests.cpp | 2 +- 4 files changed, 138 insertions(+), 16 deletions(-) diff --git a/include/TIC/DatasetExtractor.h b/include/TIC/DatasetExtractor.h index 67f65f3..5113616 100644 --- a/include/TIC/DatasetExtractor.h +++ b/include/TIC/DatasetExtractor.h @@ -50,7 +50,8 @@ class DatasetExtractor { static constexpr uint8_t LF = 0x0a; static constexpr uint8_t CR = 0x0d; static constexpr uint8_t START_MARKER = LF; /*!< Dataset start marker (line feed) */ - static constexpr uint8_t END_MARKER = CR; /*!< Dataset end marker (carriage return) */ + static constexpr uint8_t END_MARKER_TIC_1 = CR; /*!< 1st possible dataset end marker for TIC (carriage return) */ + static constexpr uint8_t END_MARKER_TIC_2 = LF; /*!< 2nd possible dataset end marker for TIC (line feed) */ static constexpr unsigned int MAX_DATASET_SIZE = 128; /*!< Max size for a dataset storage (in bytes) */ /* Methods */ diff --git a/src/TIC/DatasetExtractor.cpp b/src/TIC/DatasetExtractor.cpp index aad5f3b..70d6764 100644 --- a/src/TIC/DatasetExtractor.cpp +++ b/src/TIC/DatasetExtractor.cpp @@ -45,7 +45,20 @@ unsigned int TIC::DatasetExtractor::pushBytes(const uint8_t* buffer, unsigned in //FIXME: historical TIC uses LF, standard TIC uses CR //If we allow LD below, and we have transmission errors, we may become out of sync if we catch a wrong extraneous LF... we will be out of sync and get only empty datasets because we swap start and end markers //We should have a way to recover from this, for example by detecting 0 size datasets and thus understand we need to invert sync state - uint8_t* endOfDataset = (uint8_t*)(memchr(buffer, TIC::DatasetExtractor::LF, len)); /* Search for end of dataset */ + uint8_t* probeEndOfDataset1 = nullptr; /* Attempt to detect the end of a dataset formatted with standard TIC */ + uint8_t* probeEndOfDataset2 = nullptr; /* Attempt to detect the end of a dataset formatted with historical TIC */ + uint8_t* endOfDataset = nullptr; + probeEndOfDataset1 = (uint8_t*)(memchr(buffer, TIC::DatasetExtractor::END_MARKER_TIC_1, len)); + if (probeEndOfDataset1) { + endOfDataset = probeEndOfDataset1; + } + else { + probeEndOfDataset2 = (uint8_t*)(memchr(buffer, TIC::DatasetExtractor::END_MARKER_TIC_2, len)); /* Search for end of dataset formatted with historical TIC */ + if (probeEndOfDataset2) { + endOfDataset = probeEndOfDataset2; + } + } + if (endOfDataset) { /* We have an end of dataset marker in the buffer, we can extract the full dataset */ unsigned int leadingBytesInPreviousDataset = endOfDataset - buffer; usedBytes = this->processIncomingDatasetBytes(buffer, leadingBytesInPreviousDataset, true); /* Copy the buffer up to (but exclusing the end of dataset marker), the dataset is complete */ diff --git a/test/src/TicDatasetExtractor_tests.cpp b/test/src/TicDatasetExtractor_tests.cpp index 0333008..0424875 100644 --- a/test/src/TicDatasetExtractor_tests.cpp +++ b/test/src/TicDatasetExtractor_tests.cpp @@ -96,9 +96,9 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_dataset_10byte } } -TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes) { +TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type1) { uint8_t start_marker = TIC::DatasetExtractor::START_MARKER; - uint8_t end_marker = TIC::DatasetExtractor::END_MARKER; + uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_1; uint8_t buffer[] = { 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 }; DatasetDecoderStub stub; TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub); @@ -113,9 +113,45 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_ } } -TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes) { +TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type2) { uint8_t start_marker = TIC::DatasetExtractor::START_MARKER; - uint8_t end_marker = TIC::DatasetExtractor::END_MARKER; + uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_2; + uint8_t buffer[] = { 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 }; + DatasetDecoderStub stub; + TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub); + de.pushBytes(&start_marker, 1); + de.pushBytes(buffer, sizeof(buffer)); + de.pushBytes(&end_marker, 1); + if (stub.decodedDatasetList.size() != 1) { + FAILF("Wrong dataset count: %zu\nDatasets received:\n%s", stub.decodedDatasetList.size(), stub.toString().c_str()); + } + if (stub.decodedDatasetList[0] != std::vector({0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39})) { + FAILF("Wrong dataset decoded: %s", vectorToHexString(stub.decodedDatasetList[0]).c_str()); + } +} + +TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type1) { + uint8_t start_marker = TIC::DatasetExtractor::START_MARKER; + uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_1; + uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 }; + DatasetDecoderStub stub; + TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub); + de.pushBytes(&start_marker, 1); + for (unsigned int pos = 0; pos < sizeof(buffer); pos++) { + de.pushBytes(buffer + pos, 1); + } + de.pushBytes(&end_marker, 1); + if (stub.decodedDatasetList.size() != 1) { + FAILF("Wrong dataset count: %zu\nDatasets received:\n%s", stub.decodedDatasetList.size(), stub.toString().c_str()); + } + if (stub.decodedDatasetList[0] != std::vector({0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39})) { + FAILF("Wrong dataset decoded: %s", vectorToHexString(stub.decodedDatasetList[0]).c_str()); + } +} + +TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type2) { + uint8_t start_marker = TIC::DatasetExtractor::START_MARKER; + uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_2; uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 }; DatasetDecoderStub stub; TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub); @@ -138,11 +174,11 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_ buffer[0] = TIC::DatasetExtractor::START_MARKER; for (unsigned int pos = 1; pos < sizeof(buffer) - 1 ; pos++) { buffer[pos] = (uint8_t)(pos & 0xff); - if (buffer[pos] == TIC::DatasetExtractor::START_MARKER || buffer[pos] == TIC::DatasetExtractor::END_MARKER || buffer[pos] == TIC::Unframer::START_MARKER || buffer[pos] == TIC::Unframer::END_MARKER) { + if (buffer[pos] == TIC::DatasetExtractor::START_MARKER || buffer[pos] == TIC::DatasetExtractor::END_MARKER_TIC_1 || buffer[pos] == TIC::Unframer::START_MARKER || buffer[pos] == TIC::Unframer::END_MARKER) { buffer[pos] = 0x00; /* Remove any frame of dataset delimiters */ } } - buffer[sizeof(buffer) - 1] = TIC::DatasetExtractor::END_MARKER; + buffer[sizeof(buffer) - 1] = TIC::DatasetExtractor::END_MARKER_TIC_1; DatasetDecoderStub stub; TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub); de.pushBytes(buffer, sizeof(buffer) / 2); @@ -155,9 +191,28 @@ TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_ } } -TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves) { +TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type1) { + uint8_t start_marker = TIC::DatasetExtractor::START_MARKER; + uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_1; + uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 }; + DatasetDecoderStub stub; + TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub); + de.pushBytes(&start_marker, 1); + for (uint8_t pos = 0; pos < sizeof(buffer); pos++) { + de.pushBytes(buffer + pos, 1); + } + de.pushBytes(&end_marker, 1); + if (stub.decodedDatasetList.size() != 1) { + FAILF("Wrong dataset count: %zu\nDatasets received:\n%s", stub.decodedDatasetList.size(), stub.toString().c_str()); + } + if (stub.decodedDatasetList[0] != std::vector({0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39})) { + FAILF("Wrong dataset decoded: %s", vectorToHexString(stub.decodedDatasetList[0]).c_str()); + } +} + +TEST(TicDatasetExtractor_tests, TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type2) { uint8_t start_marker = TIC::DatasetExtractor::START_MARKER; - uint8_t end_marker = TIC::DatasetExtractor::END_MARKER; + uint8_t end_marker = TIC::DatasetExtractor::END_MARKER_TIC_2; uint8_t buffer[] = { 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39 }; DatasetDecoderStub stub; TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub); @@ -194,7 +249,7 @@ static void TicUnframer_test_file_sent_by_chunks(const std::vector& tic } TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_historical_TIC) { - std::vector ticData = readVectorFromDisk("./samples/continuous_linky_3P_historical_TIC_2024_sample.bin"); + std::vector ticData = readVectorFromDisk("./samples/continuous_linky_3P_historical_TIC_sample.bin"); for (unsigned int chunkSize = 1; chunkSize <= TIC::DatasetExtractor::MAX_DATASET_SIZE; chunkSize++) { DatasetDecoderStub stub; @@ -219,7 +274,56 @@ TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_historical_TIC) expectedTotalDatasetCount += 6; /* 6 more trailing dataset within an unterminated frame */ #endif - if (stub.decodedDatasetList.size() != expectedTotalDatasetCount) { + if (stub.decodedDatasetList.size() != expectedTotalDatasetCount) { + FAILF("When using chunk size %u: Wrong dataset count: %zu, expected %zu\nDatasets received:\n%s", chunkSize, stub.decodedDatasetList.size(), expectedTotalDatasetCount, stub.toString().c_str()); + } + char firstDatasetAsCString[] = "ADCO 056234673197 L"; + std::vector expectedFirstDatasetInFrame(firstDatasetAsCString, firstDatasetAsCString+strlen(firstDatasetAsCString)); + if (stub.decodedDatasetList[0] != expectedFirstDatasetInFrame) { + FAILF("Unexpected first dataset in first frame:\nGot: %s\nExpected: %s\n", vectorToHexString(stub.decodedDatasetList[0]).c_str(), vectorToHexString(expectedFirstDatasetInFrame).c_str()); + } + char lastDatasetAsCString[] = "PPOT 00 #"; + std::vector expectedLastDatasetInFrame(lastDatasetAsCString, lastDatasetAsCString+strlen(lastDatasetAsCString)); + if (stub.decodedDatasetList[nbExpectedDatasetPerFrame-1] != expectedLastDatasetInFrame) { + FAILF("Unexpected last dataset in first frame:\nGot: %s\nExpected: %s\n", vectorToHexString(stub.decodedDatasetList[nbExpectedDatasetPerFrame-1]).c_str(), vectorToHexString(expectedLastDatasetInFrame).c_str()); + } + for (std::size_t datasetIndex = 0; datasetIndex < stub.decodedDatasetList.size(); datasetIndex++) { + std::size_t receivedDatasetSize = stub.decodedDatasetList[datasetIndex].size(); + std::size_t expectedDatasetSize = datasetExpectedSizes[datasetIndex % nbExpectedDatasetPerFrame]; + if (receivedDatasetSize != expectedDatasetSize) { + FAILF("When using chunk size %u: Wrong dataset decoded at index %zu in frame. Expected %zu bytes, got %zu bytes. Dataset content: %s", chunkSize, datasetIndex, expectedDatasetSize, receivedDatasetSize, vectorToHexString(stub.decodedDatasetList[datasetIndex]).c_str()); + } + } + } +} + +TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_historical_TIC_2) { + std::vector ticData = readVectorFromDisk("./samples/continuous_linky_3P_historical_TIC_2024_sample.bin"); + + for (unsigned int chunkSize = 1; chunkSize <= TIC::DatasetExtractor::MAX_DATASET_SIZE; chunkSize++) { + DatasetDecoderStub stub; + TIC::DatasetExtractor de(datasetDecoderStubUnwrapInvoke, &stub); + TIC::Unframer tu(datasetExtractorUnwrapForwardFrameBytes, datasetExtractorUnWrapFrameFinished, &de); + + TicUnframer_test_file_sent_by_chunks(ticData, chunkSize, tu); + + /** + * @brief Sizes (in bytes) of the successive dataset in each repeated TIC frame + */ + std::size_t datasetExpectedSizes[] = { 19, /* ADCO label */ + 14, 11, 16, 11, + 12, 12, 12, /* Three times IINST? labels (on each phase) */ + 11, 11, 11, /* Three times IMAX? labels (on each phase) */ + 12, 12, 9, 17, 9 + }; + unsigned int nbExpectedDatasetPerFrame = sizeof(datasetExpectedSizes)/sizeof(datasetExpectedSizes[0]); + + std::size_t expectedTotalDatasetCount = 4 * nbExpectedDatasetPerFrame; /* 4 frames, each containing the above datasets */ +#ifdef __TIC_UNFRAMER_FORWARD_FRAME_BYTES_ON_THE_FLY__ + expectedTotalDatasetCount += 7; /* 7 more trailing dataset within an unterminated frame */ +#endif + + if (stub.decodedDatasetList.size() != expectedTotalDatasetCount) { FAILF("When using chunk size %u: Wrong dataset count: %zu, expected %zu\nDatasets received:\n%s", chunkSize, stub.decodedDatasetList.size(), expectedTotalDatasetCount, stub.toString().c_str()); } char firstDatasetAsCString[] = "ADCO 056234673197 L"; @@ -295,11 +399,15 @@ TEST(TicDatasetExtractor_tests, Chunked_sample_unframe_dsextract_standard_TIC) { #ifndef USE_CPPUTEST void runTicDatasetExtractorAllUnitTests() { TicDatasetExtractor_test_one_pure_dataset_10bytes(); - TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes(); - TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes(); + TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type1(); + TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_markers_10bytes_end_of_dataset_type2(); + TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type1(); + TicDatasetExtractor_test_one_pure_stx_etx_frame_standalone_bytes_end_of_dataset_type2(); TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_max_buffer(); - TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves(); + TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type1(); + TicDatasetExtractor_test_one_pure_stx_etx_frame_two_halves_end_of_dataset_type2(); Chunked_sample_unframe_dsextract_historical_TIC(); + Chunked_sample_unframe_dsextract_historical_TIC_2(); Chunked_sample_unframe_dsextract_standard_TIC(); } #endif // USE_CPPUTEST diff --git a/test/src/TicDatasetView_tests.cpp b/test/src/TicDatasetView_tests.cpp index 589de86..c4b7b53 100644 --- a/test/src/TicDatasetView_tests.cpp +++ b/test/src/TicDatasetView_tests.cpp @@ -215,7 +215,7 @@ TEST(TicDatasetView_tests, TicDatasetView_extra_leading_start_marker) { TEST(TicDatasetView_tests, TicDatasetView_extra_trailing_end_marker) { char dataset[] = { "ADCO 012345678901 E*"}; - dataset[sizeof(dataset)-1-1] = TIC::DatasetExtractor::END_MARKER; /* Replace the * with our end marker (-1 to get inside the buffer, -1 again to move before the terminating '\0') */ + dataset[sizeof(dataset)-1-1] = TIC::DatasetExtractor::END_MARKER_TIC_1; /* Replace the * with our end marker (-1 to get inside the buffer, -1 again to move before the terminating '\0') */ const uint8_t* datasetBuf = reinterpret_cast(dataset);