From c2422820cab8331fe8680d42a2275ef7587057ea Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 14 Jan 2025 09:27:21 +0100 Subject: [PATCH 1/8] [uss_qualifier/scenarios/netrid/nominal_behavior] specify check severity in test scenario documentation only (contrib #404) --- .../astm/netrid/common/nominal_behavior.py | 2 -- .../astm/netrid/display_data_evaluator.py | 12 -------- .../scenarios/astm/netrid/injection.py | 3 -- .../scenarios/astm/netrid/v19/misbehavior.md | 4 +-- .../astm/netrid/v19/nominal_behavior.md | 30 +++++++++---------- .../scenarios/astm/netrid/v22a/misbehavior.md | 4 +-- .../astm/netrid/v22a/nominal_behavior.md | 30 +++++++++---------- 7 files changed, 34 insertions(+), 51 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py index d98e120c2a..403e5faa99 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py @@ -5,7 +5,6 @@ from monitoring.monitorlib.errors import stacktrace_string from monitoring.monitorlib.rid import RIDVersion -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource from monitoring.uss_qualifier.resources.netrid import ( FlightDataResource, @@ -139,7 +138,6 @@ def cleanup(self): stacktrace = stacktrace_string(e) check.record_failed( summary="Error while trying to delete test flight", - severity=Severity.Medium, details=f"While trying to delete a test flight from {sp.participant_id}, encountered error:\n{stacktrace}", ) self.end_cleanup() diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py index b63b337aa9..f0b9c720af 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py @@ -325,7 +325,6 @@ def _evaluate_observation( check.record_failed( summary="Observation failed", details=f"When queried for an observation in {_rect_str(rect)}, {observer.participant_id} returned code {query.status_code}", - severity=Severity.Medium, query_timestamps=[query.request.timestamp], ) return @@ -362,7 +361,6 @@ def _evaluate_normal_observation( if duplicates: check.record_failed( "Duplicate flight IDs in observation", - Severity.Medium, details="Duplicate flight IDs observed: " + ", ".join(duplicates), query_timestamps=[query.request.timestamp], ) @@ -442,7 +440,6 @@ def _evaluate_normal_observation( ): check.record_failed( "Position Data is using extrapolation when Telemetry is available.", - Severity.Medium, details=( f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID " f"{mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} " @@ -472,7 +469,6 @@ def _evaluate_normal_observation( check.record_failed( summary=f"Observation of details failed for {mapping.observed_flight.id}", details=f"When queried for details of observation (ID {mapping.observed_flight.id}), {observer.participant_id} returned code {query.status_code}", - severity=Severity.Medium, query_timestamps=[query.request.timestamp], ) else: @@ -601,7 +597,6 @@ def _evaluate_area_too_large_observation( check.record_failed( summary="Did not receive expected error code for too-large area request", details=f"{observer.participant_id} was queried for flights in {_rect_str(rect)} with a diagonal of {diagonal} which is larger than the maximum allowed diagonal of {self._rid_version.max_diagonal_km}. The expected error code is 413, but instead code {query.status_code} was received.", - severity=Severity.High, query_timestamps=[query.request.timestamp], ) @@ -853,7 +848,6 @@ def _evaluate_sp_observation( if not sp_observation.dss_isa_query.success: check.record_failed( summary="Could not query ISAs from DSS", - severity=Severity.Medium, details=f"Query to {self._dss.participant_id}'s DSS at {sp_observation.dss_isa_query.query.request.url} failed {sp_observation.dss_isa_query.query.status_code}", query_timestamps=[ sp_observation.dss_isa_query.query.request.initiated_at.datetime @@ -918,7 +912,6 @@ def _evaluate_normal_sp_observation( if errors: check.record_failed( summary="/flights response failed schema validation", - severity=Severity.Medium, details="The response received from querying the /flights endpoint failed validation against the required OpenAPI schema:\n" + "\n".join( f"At {e.json_path} in the response: {e.message}" @@ -1179,7 +1172,6 @@ def _evaluate_normal_sp_observation( if not details_query.success: check.record_failed( summary="Flight details query not successful", - severity=Severity.Medium, details=f"Flight details query to {details_query.query.request.url} failed {details_query.status_code}", query_timestamps=[details_query.query.request.timestamp], ) @@ -1195,7 +1187,6 @@ def _evaluate_normal_sp_observation( if errors: check.record_failed( summary="Flight details response failed schema validation", - severity=Severity.Medium, details="The response received from querying the flight details endpoint failed validation against the required OpenAPI schema:\n" + "\n".join( f"At {e.json_path} in the response: {e.message}" @@ -1223,7 +1214,6 @@ def _evaluate_area_too_large_sp_observation( check.record_failed( summary="Flight discovered using too-large area request", details=f"{mapping.injected_flight.uss_participant_id} was queried for flights in {_rect_str(rect)} with a diagonal of {diagonal} km which is larger than the maximum allowed diagonal of {self._rid_version.max_diagonal_km} km. The expected error code is 413, but instead a valid response containing the expected flight was received.", - severity=Severity.High, query_timestamps=[ mapping.observed_flight.query.query.request.timestamp ], @@ -1243,7 +1233,6 @@ def _evaluate_sp_flight_recent_positions_times( check.record_failed( "A Position timestamp was older than the tolerance.", details=f"Position timestamp: {p.time}, query time: {query_time}", - severity=Severity.Medium, ) def _evaluate_sp_flight_recent_positions_crossing_area_boundary( @@ -1258,7 +1247,6 @@ def fail_check(): check.record_failed( "A position outside the area was neither preceded nor followed by a position inside the area.", details=f"Positions: {f.recent_positions}, requested_area: {requested_area}", - severity=Severity.Medium, ) positions = _chronological_positions(f) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py b/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py index ba55748db7..11460b3fbf 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py @@ -60,7 +60,6 @@ def inject_flights( if query.status_code != 200: check.record_failed( summary="Error while trying to inject test flight", - severity=Severity.High, details=f"Expected response code 200 from {target.participant_id} but received {query.status_code} while trying to inject a test flight", query_timestamps=[query.request.timestamp], ) @@ -68,7 +67,6 @@ def inject_flights( if "json" not in query.response or query.response.json is None: check.record_failed( summary="Response to test flight injection request did not contain a JSON body", - severity=Severity.High, details=f"Expected a JSON body in response to flight injection request", query_timestamps=[query.request.timestamp], ) @@ -115,7 +113,6 @@ def inject_flights( if errors: check.record_failed( summary="Injected flights not suitable for test", - severity=Severity.High, details="When checking the suitability of the flights (as injected) for the test, found:\n" + "\n".join(errors), query_timestamps=[f.query_timestamp for f in injected_flights], diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md index af43f8b57f..dcff5390e0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md @@ -28,14 +28,14 @@ A [`DSSInstanceResource`](../../../../resources/astm/f3411/dss.py) is required f In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. -#### Successful injection check +#### 🛑Successful injection check Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. **[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)** requires a Service Provider to provide a persistently supported test instance of their implementation. This check will fail if the flight was not successfully injected. -#### Identifiable flights check +#### 🛑Identifiable flights check This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md index 6a94735fcb..d6b4c71750 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md @@ -32,7 +32,7 @@ If specified, uss_qualifier will act as a Display Provider and check a DSS insta In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. -#### Successful injection check +#### 🛑Successful injection check Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. @@ -46,7 +46,7 @@ TODO: Validate injected flights, especially to make sure they contain the specif Per **[interuss.automated_testing.rid.injection.UpsertTestResult](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the NetRID Service Provider under test should only make valid modifications to the injected flights. This includes: * A flight with the specified injection ID must be returned. -#### Identifiable flights check +#### 🛑Identifiable flights check This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. @@ -54,37 +54,37 @@ This particular test requires each flight to be uniquely identifiable by its 2D If a DSS was provided to this test scenario, uss_qualifier acts as a Display Provider to query Service Providers under test in this step. -#### ISA query check +#### ⚠️ISA query check **[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. If uss_qualifier is unable to query the DSS for ISAs, this check will fail. -#### Area too large check +#### 🛑Area too large check **[astm.f3411.v19.NET0250](../../../../requirements/astm/f3411/v19.md)** requires that a NetRID Service Provider rejects a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. #### [Flight presence checks](./display_data_evaluator_flight_presence.md) -#### Flights data format check +#### ⚠️Flights data format check **[astm.f3411.v19.NET0710,1](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the P2P portion of the OpenAPI specification. This check will fail if the response to the /flights endpoint does not validate against the OpenAPI-specified schema. #### [Flight consistency with Common Data Dictionary checks](./common_dictionary_evaluator_sp_flight.md) -#### Recent positions timestamps check +#### ⚠️Recent positions timestamps check **[astm.f3411.v19.NET0270](../../../../requirements/astm/f3411/v19.md)** requires all recent positions to be within the NetMaxNearRealTimeDataPeriod. This check will fail if any of the reported positions are older than the maximally allowed period plus NetSpDataResponseTime99thPercentile. -#### Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing check +#### ⚠️Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing check **[astm.f3411.v19.NET0270](../../../../requirements/astm/f3411/v19.md)** requires that when an aircraft enters or leaves the queried area, the last or first reported position outside the area is provided in the recent positions, as long as it is not older than NetMaxNearRealTimeDataPeriod. This implies that any recent position outside the area must be either preceded or followed by a position inside the area. (This check validates NET0270 b and c). -#### Successful flight details query check +#### ⚠️Successful flight details query check **[astm.f3411.v19.NET0710,2](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. -#### Flight details data format check +#### ⚠️Flight details data format check **[astm.f3411.v19.NET0710,2](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the P2P portion of the OpenAPI specification. This check will fail if the response to the flight details endpoint does not validate against the OpenAPI-specified schema. @@ -92,17 +92,17 @@ This implies that any recent position outside the area must be either preceded o In this step, all observers are queried for the flights they observe. Based on the known flights that were injected into the SPs in the first step, these observations are checked against expected behavior/data. Observation rectangles are chosen to encompass the known flights when possible. -#### Area too large check +#### 🛑Area too large check **[astm.f3411.v19.NET0430](../../../../requirements/astm/f3411/v19.md)** require that a NetRID Display Provider reject a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. -#### Successful observation check +#### ⚠️Successful observation check Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call to each observer is expected to succeed since a valid view was provided by uss_qualifier. #### [Clustering checks](./display_data_evaluator_clustering.md) -#### Duplicate flights check +#### ⚠️Duplicate flights check Per **[interuss.automated_testing.rid.observation.UniqueFlights](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the same flight ID may not be reported by a Display Provider for two flights in the same observation. @@ -110,11 +110,11 @@ Per **[interuss.automated_testing.rid.observation.UniqueFlights](../../../../req #### [Flight consistency with Common Data Dictionary checks](./common_dictionary_evaluator_dp_flight.md) -#### Telemetry being used when present check +#### ⚠️Telemetry being used when present check **[astm.f3411.v19.NET0290](../../../../requirements/astm/f3411/v19.md)** requires a SP uses Telemetry vs extrapolation when telemetry is present. -#### Successful details observation check +#### ⚠️Successful details observation check Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call for flight details is expected to succeed since a valid ID was provided by uss_qualifier. @@ -122,6 +122,6 @@ Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../. The cleanup phase of this test scenario attempts to remove injected data from all SPs. -### Successful test deletion check +### ⚠️Successful test deletion check **[interuss.automated_testing.rid.injection.DeleteTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md index 698084d61d..46144925b0 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md @@ -28,14 +28,14 @@ A [`DSSInstanceResource`](../../../../resources/astm/f3411/dss.py) is required f In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. -#### Successful injection check +#### 🛑Successful injection check Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. **[astm.f3411.v22a.NET0500](../../../../requirements/astm/f3411/v22a.md)** requires a Service Provider to provide a persistently supported test instance of their implementation. This check will fail if the flight was not successfully injected. -#### Identifiable flights check +#### 🛑Identifiable flights check This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md index 6b99cba9dc..4883b80654 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md @@ -32,7 +32,7 @@ If specified, uss_qualifier will act as a Display Provider and check a DSS insta In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. -#### Successful injection check +#### 🛑Successful injection check Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. @@ -46,7 +46,7 @@ TODO: Validate injected flights, especially to make sure they contain the specif Per **[interuss.automated_testing.rid.injection.UpsertTestResult](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the NetRID Service Provider under test should only make valid modifications to the injected flights. This includes: * A flight with the specified injection ID must be returned. -#### Identifiable flights check +#### 🛑Identifiable flights check This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. @@ -54,37 +54,37 @@ This particular test requires each flight to be uniquely identifiable by its 2D If a DSS was provided to this test scenario, uss_qualifier acts as a Display Provider to query Service Providers under test in this step. -#### ISA query check +#### ⚠️ISA query check **[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. If uss_qualifier is unable to query the DSS for ISAs, this check will fail. -#### Area too large check +#### 🛑Area too large check **[astm.f3411.v22a.NET0250](../../../../requirements/astm/f3411/v22a.md)** requires that a NetRID Service Provider rejects a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. #### [Flight presence checks](./display_data_evaluator_flight_presence.md) -#### Flights data format check +#### ⚠️Flights data format check **[astm.f3411.v22a.NET0710,1](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md)** requires a Service Provider to implement the P2P portion of the OpenAPI specification. This check will fail if the response to the /flights endpoint does not validate against the OpenAPI-specified schema. #### [Flight consistency with Common Data Dictionary checks](./common_dictionary_evaluator_sp_flight.md) -#### Recent positions timestamps check +#### ⚠️Recent positions timestamps check **[astm.f3411.v22a.NET0270](../../../../requirements/astm/f3411/v22a.md)** requires all recent positions to be within the NetMaxNearRealTimeDataPeriod. This check will fail if any of the reported positions are older than the maximally allowed period plus NetSpDataResponseTime99thPercentile. -#### Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing check +#### ⚠️Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing check **[astm.f3411.v22a.NET0270](../../../../requirements/astm/f3411/v22a.md)** requires that when an aircraft enters or leaves the queried area, the last or first reported position outside the area is provided in the recent positions, as long as it is not older than NetMaxNearRealTimeDataPeriod. This implies that any recent position outside the area must be either preceded or followed by a position inside the area. (This check validates NET0270 b and c). -#### Successful flight details query check +#### ⚠️Successful flight details query check **[astm.f3411.v22a.NET0710,2](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. -#### Flight details data format check +#### ⚠️Flight details data format check **[astm.f3411.v22a.NET0710,2](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the P2P portion of the OpenAPI specification. This check will fail if the response to the flight details endpoint does not validate against the OpenAPI-specified schema. @@ -94,17 +94,17 @@ This implies that any recent position outside the area must be either preceded o In this step, all observers are queried for the flights they observe. Based on the known flights that were injected into the SPs in the first step, these observations are checked against expected behavior/data. Observation rectangles are chosen to encompass the known flights when possible. -#### Area too large check +#### 🛑Area too large check **[astm.f3411.v22a.NET0430](../../../../requirements/astm/f3411/v22a.md)** require that a NetRID Display Provider reject a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. -#### Successful observation check +#### ⚠️Successful observation check Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call to each observer is expected to succeed since a valid view was provided by uss_qualifier. #### [Clustering checks](./display_data_evaluator_clustering.md) -#### Duplicate flights check +#### ⚠️Duplicate flights check Per **[interuss.automated_testing.rid.observation.UniqueFlights](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the same flight ID may not be reported by a Display Provider for two flights in the same observation. @@ -112,11 +112,11 @@ Per **[interuss.automated_testing.rid.observation.UniqueFlights](../../../../req #### [Flight consistency with Common Data Dictionary checks](./common_dictionary_evaluator_dp_flight.md) -#### Telemetry being used when present check +#### ⚠️Telemetry being used when present check **[astm.f3411.v22a.NET0290](../../../../requirements/astm/f3411/v22a.md)** requires a SP uses Telemetry vs extrapolation when telemetry is present. -#### Successful details observation check +#### ⚠️Successful details observation check Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call for flight details is expected to succeed since a valid ID was provided by uss_qualifier. @@ -126,6 +126,6 @@ Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../. The cleanup phase of this test scenario attempts to remove injected data from all SPs. -### Successful test deletion check +### ⚠️Successful test deletion check **[interuss.automated_testing.rid.injection.DeleteTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)** From 455389896aaf72dfdbe554d187ea15dfcbb6b165 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 14 Jan 2025 09:35:13 +0100 Subject: [PATCH 2/8] [uss_qualifier/scenarios/netrid/dss/heavy_traffic_concurrent] specify check severity in test scenario documentation only (contrib #404) --- .../common/dss/heavy_traffic_concurrent.py | 7 ------- .../netrid/v19/dss/heavy_traffic_concurrent.md | 16 ++++++++-------- .../netrid/v22a/dss/heavy_traffic_concurrent.md | 16 ++++++++-------- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py index c4c912efb6..28f9061120 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py @@ -19,7 +19,6 @@ from monitoring.monitorlib.mutate.rid import ChangedISA from monitoring.monitorlib.rid import RIDVersion from monitoring.prober.infrastructure import register_resource_type -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource @@ -163,7 +162,6 @@ def _get_isas_by_id_concurrent_step(self): if fetched_isa.status_code != 200: main_check.record_failed( f"ISA retrieval query failed for {isa_id}", - severity=Severity.High, details=f"ISA retrieval query for {isa_id} yielded code {fetched_isa.status_code}", ) @@ -312,7 +310,6 @@ def _create_isas_concurrent_step(self): if changed_isa.query.response.code != 200: main_check.record_failed( f"ISA creation failed for {isa_id}", - severity=Severity.High, details=f"ISA creation for {isa_id} returned {changed_isa.query.response.code}", ) else: @@ -347,7 +344,6 @@ def _search_area_step(self): if isa_id not in isas.isas.keys(): sub_check.record_failed( f"ISAs search did not return ISA {isa_id} that was just created", - severity=Severity.High, details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()} and is missing some of the created ISAs", query_timestamps=[isas.dss_query.query.request.timestamp], ) @@ -387,7 +383,6 @@ def _delete_isas(self): if deleted_isa.query.response.code != 200: main_check.record_failed( f"ISA deletion failed for {isa_id}", - severity=Severity.High, details=f"ISA deletion for {isa_id} returned {deleted_isa.query.response.code}", ) @@ -421,7 +416,6 @@ def _get_deleted_isas(self): if fetched_isa.status_code != 404: check.record_failed( f"ISA retrieval succeeded for {isa_id}", - severity=Severity.High, details=f"ISA retrieval for {isa_id} returned {fetched_isa.status_code}", query_timestamps=[fetched_isa.query.request.timestamp], ) @@ -442,7 +436,6 @@ def _search_deleted_isas(self): if isa_id in isas.isas.keys(): check.record_failed( f"ISAs search returned deleted ISA {isa_id}", - severity=Severity.High, details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()} that contained some of the ISAs we had previously deleted.", query_timestamps=[isas.dss_query.query.request.timestamp], ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md index 86c48384b3..ffed504e64 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md @@ -39,7 +39,7 @@ This test case will: This step attempts to concurrently create multiple ISAs, as specified in this scenario's resource, at the configured DSS. -#### Concurrent ISAs creation check +#### 🛑Concurrent ISAs creation check If any of the concurrent ISA creation requests fail or leads to the creation of an incorrect ISA, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. @@ -47,7 +47,7 @@ If any of the concurrent ISA creation requests fail or leads to the creation of This step attempts to concurrently retrieve the previously created ISAs from the DSS. -#### Successful Concurrent ISA query check +#### 🛑Successful Concurrent ISA query check If any of the ISAs cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. @@ -55,11 +55,11 @@ If any of the ISAs cannot be queried, the GET ISA DSS endpoint in **[interuss.f3 This test step searches the area in which the ISAs were concurrently created, and expects to find all of them. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### Correct ISAs returned by search check +#### 🛑Correct ISAs returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. @@ -67,7 +67,7 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts to concurrently delete the earlier created ISAs. -#### ISAs deletion query success check +#### 🛑ISAs deletion query success check If an ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. @@ -75,7 +75,7 @@ If an ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,b This step attempts to concurrently access the previously deleted ISAs from the DSS. -#### ISAs not found check +#### 🛑ISAs not found check The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -83,11 +83,11 @@ The ISA fetch request was about a deleted ISA, as such the DSS should reject it This step issues a search for active ISAs in the area of the previously deleted ISAs from the DSS. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISAs not returned by search check +#### 🛑ISAs not returned by search check The ISA search area parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md index eb2b486b0a..e2f8b6e6ab 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md @@ -39,7 +39,7 @@ This test case will: This step attempts to concurrently create multiple ISAs, as specified in this scenario's resource, at the configured DSS. -#### Concurrent ISAs creation check +#### 🛑Concurrent ISAs creation check If any of the concurrent ISA creation requests fail or leads to the creation of an incorrect ISA, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. @@ -47,7 +47,7 @@ If any of the concurrent ISA creation requests fail or leads to the creation of This step attempts to concurrently retrieve the previously created ISAs from the DSS. -#### Successful Concurrent ISA query check +#### 🛑Successful Concurrent ISA query check If any of the ISAs cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. @@ -55,11 +55,11 @@ If any of the ISAs cannot be queried, the GET ISA DSS endpoint in **[interuss.f3 This test step searches the area in which the ISAs were concurrently created, and expects to find all of them. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### Correct ISAs returned by search check +#### 🛑Correct ISAs returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. @@ -67,7 +67,7 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts to concurrently delete the earlier created ISAs. -#### ISAs deletion query success check +#### 🛑ISAs deletion query success check If an ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. @@ -75,7 +75,7 @@ If an ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030, This step attempts to concurrently access the previously deleted ISAs from the DSS. -#### ISAs not found check +#### 🛑ISAs not found check The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -83,11 +83,11 @@ The ISA fetch request was about a deleted ISA, as such the DSS should reject it This step issues a search for active ISAs in the area of the previously deleted ISAs from the DSS. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISAs not returned by search check +#### 🛑ISAs not returned by search check The ISA search area parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. From 891455c047c0ca1e1434223d774d6686160d64c5 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 14 Jan 2025 09:40:04 +0100 Subject: [PATCH 3/8] [uss_qualifier/scenarios/netrid/injection] specify check severity in test scenario documentation only (contrib #404) --- monitoring/uss_qualifier/scenarios/astm/netrid/injection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py b/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py index 11460b3fbf..4520e8b839 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/injection.py @@ -12,7 +12,6 @@ TestFlight, CreateTestParameters, ) -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.netrid import ( FlightDataResource, NetRIDServiceProviders, From 9a84926faef912e09460b46d3f214833d5ebe730 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 14 Jan 2025 10:03:39 +0100 Subject: [PATCH 4/8] [uss_qualifier/scenarios/netrid/dss/isa_simple] specify check severity in test scenario documentation only (contrib #404) --- .../astm/netrid/common/dss/isa_simple.py | 19 ++----- .../scenarios/astm/netrid/dss_wrapper.py | 1 - .../astm/netrid/v19/dss/isa_simple.md | 56 +++++++++---------- .../astm/netrid/v19/dss/test_steps/put_isa.md | 2 +- .../astm/netrid/v22a/dss/isa_simple.md | 56 +++++++++---------- .../netrid/v22a/dss/test_steps/put_isa.md | 2 +- 6 files changed, 62 insertions(+), 74 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py index e36c710720..c002904ff9 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py @@ -7,7 +7,6 @@ from monitoring.monitorlib.fetch import rid as fetch from monitoring.monitorlib.mutate import rid as mutate from monitoring.prober.infrastructure import register_resource_type -from monitoring.uss_qualifier.common_data_definitions import Severity from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource @@ -87,8 +86,7 @@ def _delete_isa_if_exists(self): if not fetched.success and fetched.status_code != 404: check.record_failed( "ISA information could not be retrieved", - Severity.High, - f"{self._dss.participant_id} DSS instance returned {fetched.status_code} when queried for ISA {self._isa_id}", + details=f"{self._dss.participant_id} DSS instance returned {fetched.status_code} when queried for ISA {self._isa_id}", query_timestamps=[fetched.query.request.timestamp], ) @@ -109,8 +107,7 @@ def _delete_isa_if_exists(self): if not deleted.dss_query.success: check.record_failed( "Could not delete pre-existing ISA", - Severity.High, - f"Attempting to delete ISA {self._isa_id} from the {self._dss.participant_id} DSS returned error {deleted.dss_query.status_code}", + details=f"Attempting to delete ISA {self._isa_id} from the {self._dss.participant_id} DSS returned error {deleted.dss_query.status_code}", query_timestamps=[deleted.dss_query.query.request.timestamp], ) for subscriber_url, notification in deleted.notifications.items(): @@ -127,8 +124,7 @@ def _delete_isa_if_exists(self): if not notification.success: check.record_failed( "Could not notify ISA subscriber", - Severity.Medium, - f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}", + details=f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}", query_timestamps=[notification.query.request.timestamp], ) @@ -149,8 +145,7 @@ def _get_isa_by_id_step(self): ): check.record_failed( "DSS returned ISA with incorrect version", - Severity.High, - f"DSS should have returned an ISA with the version {self._isa_version}, but instead the ISA returned had the version {fetched.isa.version}", + details=f"DSS should have returned an ISA with the version {self._isa_version}, but instead the ISA returned had the version {fetched.isa.version}", query_timestamps=[fetched.query.request.timestamp], ) @@ -230,7 +225,6 @@ def _search_earliest_incl_step(): if self._isa_id not in isas.isas.keys(): check.record_failed( f"ISAs search did not return expected ISA {self._isa_id}", - severity=Severity.High, details=f"Search in area {self._isa_area} from time {earliest} returned ISAs {isas.isas.keys()}", query_timestamps=[isas.dss_query.query.request.timestamp], ) @@ -258,7 +252,6 @@ def _search_earliest_excl_step(): if self._isa_id in isas.isas.keys(): check.record_failed( f"ISAs search returned unexpected ISA {self._isa_id}", - severity=Severity.High, details=f"Search in area {self._isa_area} from time {earliest} returned ISAs {isas.isas.keys()}", query_timestamps=[isas.dss_query.query.request.timestamp], ) @@ -286,7 +279,6 @@ def _search_latest_incl_step(): if self._isa_id not in isas.isas.keys(): check.record_failed( f"ISAs search did not return expected ISA {self._isa_id}", - severity=Severity.High, details=f"Search in area {self._isa_area} to time {latest} returned ISAs {isas.isas.keys()}", query_timestamps=[isas.dss_query.query.request.timestamp], ) @@ -314,7 +306,6 @@ def _search_latest_excl_step(): if self._isa_id in isas.isas.keys(): check.record_failed( f"ISAs search returned unexpected ISA {self._isa_id}", - severity=Severity.High, details=f"Search in area {self._isa_area} to time {latest} returned ISAs {isas.isas.keys()}", query_timestamps=[isas.dss_query.query.request.timestamp], ) @@ -340,7 +331,6 @@ def _search_area_only_step(): if self._isa_id not in isas.isas.keys(): check.record_failed( f"ISAs search did not return expected ISA {self._isa_id}", - severity=Severity.High, details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()}", query_timestamps=[isas.dss_query.query.request.timestamp], ) @@ -483,7 +473,6 @@ def _search_isa_step(): if self._isa_id in isas.isas.keys(): check.record_failed( f"ISAs search returned deleted ISA {self._isa_id}", - severity=Severity.High, details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()}", query_timestamps=[isas.dss_query.query.request.timestamp], ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py index 544ee690d5..dcf7a03e14 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py @@ -360,7 +360,6 @@ def put_isa( if mutated_isa.dss_query.query.status_code == 201: sub_check.record_failed( summary=f"PUT ISA returned technically-incorrect 201", - severity=Severity.Low, details="DSS should return 200 from PUT ISA, but instead returned the reasonable-but-technically-incorrect code 201", query_timestamps=[t_dss], ) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md index ea344782c8..7a8639a48a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md @@ -36,7 +36,7 @@ part of the test. This step attempts to query the configured DSS with the ISA provided as a resource. -#### ISA created check +#### 🛑ISA created check If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. @@ -44,13 +44,13 @@ If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030, This step attempts to retrieve the previously created ISA from the DSS. -#### Successful ISA query check +#### 🛑Successful ISA query check If the ISA cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -#### ISA version match check +#### 🛑ISA version match check The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after creation, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. @@ -60,7 +60,7 @@ The DSS returns the version of the ISA in the response body. If this version do This step attempts to update the configured DSS with the ISA provided as a resource, with a slightly different end time. -#### ISA updated check +#### 🛑ISA updated check If the ISA cannot be updated, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. @@ -68,13 +68,13 @@ If the ISA cannot be updated, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030, This step attempts to retrieve at the DSS the ISA just updated. -#### Successful ISA query check +#### 🛑Successful ISA query check If the ISA cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -#### ISA version match check +#### 🛑ISA version match check The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after update, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. @@ -82,11 +82,11 @@ The DSS returns the version of the ISA in the response body. If this version do This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that overlaps with the resource ISA. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA returned by search check +#### 🛑ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. @@ -94,11 +94,11 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that does not overlap with the resource ISA. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA not returned by search check +#### 🛑ISA not returned by search check The ISA search are parameter cover the resource ISA but the earliest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -106,11 +106,11 @@ The ISA search are parameter cover the resource ISA but the earliest time does n This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that overlaps with the resource ISA. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA returned by search check +#### 🛑ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -118,11 +118,11 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that does not overlap with the resource ISA. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA not returned by search check +#### 🛑ISA not returned by search check The ISA search are parameter cover the resource ISA but the latest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -130,11 +130,11 @@ The ISA search are parameter cover the resource ISA but the latest time does not This step attempts an ISA search at the DSS with only the area of the ISA resource. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA returned by search check +#### 🛑ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -142,7 +142,7 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with an empty search area. -#### Search request rejected check +#### 🛑Search request rejected check The search request contained invalid parameters (empty search area), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -150,7 +150,7 @@ The search request contained invalid parameters (empty search area), as such the This step attempts an ISA search at the DSS with a too large search area. -#### Search request rejected check +#### 🛑Search request rejected check The search request contained invalid parameters (too large search area), as such the DSS should reject it with a 413 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -158,7 +158,7 @@ The search request contained invalid parameters (too large search area), as such This step attempts an ISA search at the DSS with a polygon defining the area that forms a loop. -#### Search request rejected check +#### 🛑Search request rejected check The search request contained invalid parameters (area polygon is a loop, which is not allowed), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -168,7 +168,7 @@ The search request contained invalid parameters (area polygon is a loop, which i This step attempts an ISA deletion with a wrong version. -#### Delete request rejected check +#### 🛑Delete request rejected check The deletion request contained invalid parameters (wrong version), as such the DSS should reject it with a 409 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)**. @@ -176,7 +176,7 @@ The deletion request contained invalid parameters (wrong version), as such the D This step attempts an ISA deletion with an empty version. -#### Delete request rejected check +#### 🛑Delete request rejected check The deletion request contained invalid parameters (empty version), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)**. @@ -184,7 +184,7 @@ The deletion request contained invalid parameters (empty version), as such the D This step attempts an ISA deletion at the DSS. -#### ISA deleted check +#### 🛑ISA deleted check If the ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. @@ -192,7 +192,7 @@ If the ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030, This step attempts to retrieve at the DSS the ISA just deleted. -#### ISA not found check +#### 🛑ISA not found check The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -200,11 +200,11 @@ The ISA fetch request was about a deleted ISA, as such the DSS should reject it This step attempts an ISA search at the DSS with only the area of the ISA resource. Since it has just been deleted, the ISA should not be returned. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA not returned by search check +#### 🛑ISA not returned by search check The ISA search are parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -212,14 +212,14 @@ The ISA search are parameter cover the resource ISA, but it has been previously The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -### Successful ISA query check +### 🛑Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### Removed pre-existing ISA check +### 🛑Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. -### Notified subscriber check +### ⚠️Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md index 17be55b1b3..e1dcfefa9b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md @@ -3,7 +3,7 @@ This page describes the content of a common test step where a creation or an update of an ISA should be successful. See `DSSWrapper.put_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). -## ISA response code check +## ℹ️ISA response code check The API for **[astm.f3411.v19.DSS0030,a](../../../../../../requirements/astm/f3411/v19.md)** specifies that the code of successful DSS responses is 200. While tolerated in some cases, if the DSS responds with an HTTP code 201 for success, this check will fail with a low severity. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md index 065792d579..2328796f68 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md @@ -36,7 +36,7 @@ part of the test. This step attempts to query the configured DSS with the ISA provided as a resource. -#### ISA created check +#### 🛑ISA created check If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. @@ -44,13 +44,13 @@ If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030 This step attempts to retrieve the previously created ISA from the DSS. -#### Successful ISA query check +#### 🛑Successful ISA query check If the ISA cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -#### ISA version match check +#### 🛑ISA version match check The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after creation, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. @@ -60,7 +60,7 @@ The DSS returns the version of the ISA in the response body. If this version do This step attempts to update the configured DSS with the ISA provided as a resource, with a slightly different end time. -#### ISA updated check +#### 🛑ISA updated check If the ISA cannot be updated, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. @@ -68,13 +68,13 @@ If the ISA cannot be updated, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030 This step attempts to retrieve at the DSS the ISA just updated. -#### Successful ISA query check +#### 🛑Successful ISA query check If the ISA cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -#### ISA version match check +#### 🛑ISA version match check The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after update, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. @@ -82,11 +82,11 @@ The DSS returns the version of the ISA in the response body. If this version do This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that overlaps with the resource ISA. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA returned by search check +#### 🛑ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. @@ -94,11 +94,11 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that does not overlap with the resource ISA. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA not returned by search check +#### 🛑ISA not returned by search check The ISA search are parameter cover the resource ISA but the earliest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -106,11 +106,11 @@ The ISA search are parameter cover the resource ISA but the earliest time does n This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that overlaps with the resource ISA. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA returned by search check +#### 🛑ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -118,11 +118,11 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that does not overlap with the resource ISA. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA not returned by search check +#### 🛑ISA not returned by search check The ISA search are parameter cover the resource ISA but the latest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -130,11 +130,11 @@ The ISA search are parameter cover the resource ISA but the latest time does not This step attempts an ISA search at the DSS with only the area of the ISA resource. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA returned by search check +#### 🛑ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -142,7 +142,7 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with an empty search area. -#### Search request rejected check +#### 🛑Search request rejected check The search request contained invalid parameters (empty search area), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -150,7 +150,7 @@ The search request contained invalid parameters (empty search area), as such the This step attempts an ISA search at the DSS with a too large search area. -#### Search request rejected check +#### 🛑Search request rejected check The search request contained invalid parameters (too large search area), as such the DSS should reject it with a 413 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -158,7 +158,7 @@ The search request contained invalid parameters (too large search area), as such This step attempts an ISA search at the DSS with a polygon defining the area that forms a loop. -#### Search request rejected check +#### 🛑Search request rejected check The search request contained invalid parameters (area polygon is a loop, which is not allowed), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -168,7 +168,7 @@ The search request contained invalid parameters (area polygon is a loop, which i This step attempts an ISA deletion with a wrong version. -#### Delete request rejected check +#### 🛑Delete request rejected check The deletion request contained invalid parameters (wrong version), as such the DSS should reject it with a 409 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)**. @@ -176,7 +176,7 @@ The deletion request contained invalid parameters (wrong version), as such the D This step attempts an ISA deletion with an empty version. -#### Delete request rejected check +#### 🛑Delete request rejected check The deletion request contained invalid parameters (empty version), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)**. @@ -184,7 +184,7 @@ The deletion request contained invalid parameters (empty version), as such the D This step attempts an ISA deletion at the DSS. -#### ISA deleted check +#### 🛑ISA deleted check If the ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. @@ -192,7 +192,7 @@ If the ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030 This step attempts to retrieve at the DSS the ISA just deleted. -#### ISA not found check +#### 🛑ISA not found check The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -200,11 +200,11 @@ The ISA fetch request was about a deleted ISA, as such the DSS should reject it This step attempts an ISA search at the DSS with only the area of the ISA resource. Since it has just been deleted, the ISA should not be returned. -#### Successful ISAs search check +#### 🛑Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### ISA not returned by search check +#### 🛑ISA not returned by search check The ISA search are parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -212,14 +212,14 @@ The ISA search are parameter cover the resource ISA, but it has been previously The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -### Successful ISA query check +### 🛑Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### Removed pre-existing ISA check +### 🛑Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. -### Notified subscriber check +### ⚠️Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md index 2605094186..ef94e45df1 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md @@ -3,7 +3,7 @@ This page describes the content of a common test step where a creation or an update of an ISA should be successful. See `DSSWrapper.put_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). -## ISA response code check +## ℹ️ISA response code check The API for **[astm.f3411.v22a.DSS0030,a](../../../../../../requirements/astm/f3411/v22a.md)** specifies that the code of successful DSS responses is 200. While tolerated in some cases, if the DSS responds with an HTTP code 201 for success, this check will fail with a low severity. From fd87973cf6d494951b17455622e87bbfcf8800c5 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 14 Jan 2025 11:33:22 +0100 Subject: [PATCH 5/8] Add default severity to undocumented checks --- monitoring/uss_qualifier/scenarios/scenario.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index 082fd9563a..dff0b98429 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -454,7 +454,10 @@ def check( check_list = ", ".join(available_checks) if self._allow_undocumented_checks: check_documentation = TestCheckDocumentation( - name=name, applicable_requirements=[], has_todo=False + name=name, + applicable_requirements=[], + has_todo=False, + severity=Severity.Medium, ) else: test_step_name = ( From 9ce33384b6d79c0f465d91dd4bcf6f898408e76e Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Tue, 14 Jan 2025 11:44:53 +0100 Subject: [PATCH 6/8] Fix emoji spacing --- .../v19/dss/heavy_traffic_concurrent.md | 16 +++--- .../astm/netrid/v19/dss/isa_simple.md | 56 +++++++++---------- .../astm/netrid/v19/dss/test_steps/put_isa.md | 2 +- .../scenarios/astm/netrid/v19/misbehavior.md | 4 +- .../astm/netrid/v19/nominal_behavior.md | 32 ++++++----- .../v22a/dss/heavy_traffic_concurrent.md | 16 +++--- .../astm/netrid/v22a/dss/isa_simple.md | 56 +++++++++---------- .../netrid/v22a/dss/test_steps/put_isa.md | 2 +- .../scenarios/astm/netrid/v22a/misbehavior.md | 4 +- .../astm/netrid/v22a/nominal_behavior.md | 32 ++++++----- 10 files changed, 112 insertions(+), 108 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md index ffed504e64..6b6d812187 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md @@ -39,7 +39,7 @@ This test case will: This step attempts to concurrently create multiple ISAs, as specified in this scenario's resource, at the configured DSS. -#### 🛑Concurrent ISAs creation check +#### 🛑 Concurrent ISAs creation check If any of the concurrent ISA creation requests fail or leads to the creation of an incorrect ISA, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. @@ -47,7 +47,7 @@ If any of the concurrent ISA creation requests fail or leads to the creation of This step attempts to concurrently retrieve the previously created ISAs from the DSS. -#### 🛑Successful Concurrent ISA query check +#### 🛑 Successful Concurrent ISA query check If any of the ISAs cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. @@ -55,11 +55,11 @@ If any of the ISAs cannot be queried, the GET ISA DSS endpoint in **[interuss.f3 This test step searches the area in which the ISAs were concurrently created, and expects to find all of them. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑Correct ISAs returned by search check +#### 🛑 Correct ISAs returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. @@ -67,7 +67,7 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts to concurrently delete the earlier created ISAs. -#### 🛑ISAs deletion query success check +#### 🛑 ISAs deletion query success check If an ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. @@ -75,7 +75,7 @@ If an ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,b This step attempts to concurrently access the previously deleted ISAs from the DSS. -#### 🛑ISAs not found check +#### 🛑 ISAs not found check The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -83,11 +83,11 @@ The ISA fetch request was about a deleted ISA, as such the DSS should reject it This step issues a search for active ISAs in the area of the previously deleted ISAs from the DSS. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISAs not returned by search check +#### 🛑 ISAs not returned by search check The ISA search area parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md index 7a8639a48a..1cce0f2f95 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md @@ -36,7 +36,7 @@ part of the test. This step attempts to query the configured DSS with the ISA provided as a resource. -#### 🛑ISA created check +#### 🛑 ISA created check If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. @@ -44,13 +44,13 @@ If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030, This step attempts to retrieve the previously created ISA from the DSS. -#### 🛑Successful ISA query check +#### 🛑 Successful ISA query check If the ISA cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -#### 🛑ISA version match check +#### 🛑 ISA version match check The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after creation, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. @@ -60,7 +60,7 @@ The DSS returns the version of the ISA in the response body. If this version do This step attempts to update the configured DSS with the ISA provided as a resource, with a slightly different end time. -#### 🛑ISA updated check +#### 🛑 ISA updated check If the ISA cannot be updated, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,a](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. @@ -68,13 +68,13 @@ If the ISA cannot be updated, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030, This step attempts to retrieve at the DSS the ISA just updated. -#### 🛑Successful ISA query check +#### 🛑 Successful ISA query check If the ISA cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -#### 🛑ISA version match check +#### 🛑 ISA version match check The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after update, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. @@ -82,11 +82,11 @@ The DSS returns the version of the ISA in the response body. If this version do This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that overlaps with the resource ISA. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA returned by search check +#### 🛑 ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. @@ -94,11 +94,11 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that does not overlap with the resource ISA. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA not returned by search check +#### 🛑 ISA not returned by search check The ISA search are parameter cover the resource ISA but the earliest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -106,11 +106,11 @@ The ISA search are parameter cover the resource ISA but the earliest time does n This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that overlaps with the resource ISA. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA returned by search check +#### 🛑 ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -118,11 +118,11 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that does not overlap with the resource ISA. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA not returned by search check +#### 🛑 ISA not returned by search check The ISA search are parameter cover the resource ISA but the latest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -130,11 +130,11 @@ The ISA search are parameter cover the resource ISA but the latest time does not This step attempts an ISA search at the DSS with only the area of the ISA resource. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA returned by search check +#### 🛑 ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -142,7 +142,7 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with an empty search area. -#### 🛑Search request rejected check +#### 🛑 Search request rejected check The search request contained invalid parameters (empty search area), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -150,7 +150,7 @@ The search request contained invalid parameters (empty search area), as such the This step attempts an ISA search at the DSS with a too large search area. -#### 🛑Search request rejected check +#### 🛑 Search request rejected check The search request contained invalid parameters (too large search area), as such the DSS should reject it with a 413 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -158,7 +158,7 @@ The search request contained invalid parameters (too large search area), as such This step attempts an ISA search at the DSS with a polygon defining the area that forms a loop. -#### 🛑Search request rejected check +#### 🛑 Search request rejected check The search request contained invalid parameters (area polygon is a loop, which is not allowed), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -168,7 +168,7 @@ The search request contained invalid parameters (area polygon is a loop, which i This step attempts an ISA deletion with a wrong version. -#### 🛑Delete request rejected check +#### 🛑 Delete request rejected check The deletion request contained invalid parameters (wrong version), as such the DSS should reject it with a 409 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)**. @@ -176,7 +176,7 @@ The deletion request contained invalid parameters (wrong version), as such the D This step attempts an ISA deletion with an empty version. -#### 🛑Delete request rejected check +#### 🛑 Delete request rejected check The deletion request contained invalid parameters (empty version), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)**. @@ -184,7 +184,7 @@ The deletion request contained invalid parameters (empty version), as such the D This step attempts an ISA deletion at the DSS. -#### 🛑ISA deleted check +#### 🛑 ISA deleted check If the ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** is likely not implemented correctly. @@ -192,7 +192,7 @@ If the ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v19.DSS0030, This step attempts to retrieve at the DSS the ISA just deleted. -#### 🛑ISA not found check +#### 🛑 ISA not found check The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -200,11 +200,11 @@ The ISA fetch request was about a deleted ISA, as such the DSS should reject it This step attempts an ISA search at the DSS with only the area of the ISA resource. Since it has just been deleted, the ISA should not be returned. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA not returned by search check +#### 🛑 ISA not returned by search check The ISA search are parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -212,14 +212,14 @@ The ISA search are parameter cover the resource ISA, but it has been previously The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -### 🛑Successful ISA query check +### 🛑 Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### 🛑Removed pre-existing ISA check +### 🛑 Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v19.DSS0030,b](../../../../../requirements/astm/f3411/v19.md)** requirement to implement the ISA deletion endpoint might not be met. -### ⚠️Notified subscriber check +### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v19.NET0730](../../../../../requirements/astm/f3411/v19.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md index e1dcfefa9b..e7857c2f8a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/test_steps/put_isa.md @@ -3,7 +3,7 @@ This page describes the content of a common test step where a creation or an update of an ISA should be successful. See `DSSWrapper.put_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). -## ℹ️ISA response code check +## ℹ️ ISA response code check The API for **[astm.f3411.v19.DSS0030,a](../../../../../../requirements/astm/f3411/v19.md)** specifies that the code of successful DSS responses is 200. While tolerated in some cases, if the DSS responds with an HTTP code 201 for success, this check will fail with a low severity. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md index dcff5390e0..f2cc5f10ee 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/misbehavior.md @@ -28,14 +28,14 @@ A [`DSSInstanceResource`](../../../../resources/astm/f3411/dss.py) is required f In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. -#### 🛑Successful injection check +#### 🛑 Successful injection check Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. **[astm.f3411.v19.NET0500](../../../../requirements/astm/f3411/v19.md)** requires a Service Provider to provide a persistently supported test instance of their implementation. This check will fail if the flight was not successfully injected. -#### 🛑Identifiable flights check +#### 🛑 Identifiable flights check This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md index d6b4c71750..c23f74046b 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md @@ -32,7 +32,7 @@ If specified, uss_qualifier will act as a Display Provider and check a DSS insta In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. -#### 🛑Successful injection check +#### 🛑 Successful injection check Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. @@ -46,7 +46,7 @@ TODO: Validate injected flights, especially to make sure they contain the specif Per **[interuss.automated_testing.rid.injection.UpsertTestResult](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the NetRID Service Provider under test should only make valid modifications to the injected flights. This includes: * A flight with the specified injection ID must be returned. -#### 🛑Identifiable flights check +#### 🛑 Identifiable flights check This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. @@ -54,37 +54,39 @@ This particular test requires each flight to be uniquely identifiable by its 2D If a DSS was provided to this test scenario, uss_qualifier acts as a Display Provider to query Service Providers under test in this step. -#### ⚠️ISA query check +#### ⚠️ ISA query check **[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. If uss_qualifier is unable to query the DSS for ISAs, this check will fail. -#### 🛑Area too large check +#### 🛑 Area too large check **[astm.f3411.v19.NET0250](../../../../requirements/astm/f3411/v19.md)** requires that a NetRID Service Provider rejects a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. #### [Flight presence checks](./display_data_evaluator_flight_presence.md) -#### ⚠️Flights data format check +#### ⚠️ Flights data format check **[astm.f3411.v19.NET0710,1](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the P2P portion of the OpenAPI specification. This check will fail if the response to the /flights endpoint does not validate against the OpenAPI-specified schema. #### [Flight consistency with Common Data Dictionary checks](./common_dictionary_evaluator_sp_flight.md) -#### ⚠️Recent positions timestamps check +#### ⚠️ Recent positions timestamps check + **[astm.f3411.v19.NET0270](../../../../requirements/astm/f3411/v19.md)** requires all recent positions to be within the NetMaxNearRealTimeDataPeriod. This check will fail if any of the reported positions are older than the maximally allowed period plus NetSpDataResponseTime99thPercentile. -#### ⚠️Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing check +#### ⚠️ Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing check + **[astm.f3411.v19.NET0270](../../../../requirements/astm/f3411/v19.md)** requires that when an aircraft enters or leaves the queried area, the last or first reported position outside the area is provided in the recent positions, as long as it is not older than NetMaxNearRealTimeDataPeriod. This implies that any recent position outside the area must be either preceded or followed by a position inside the area. (This check validates NET0270 b and c). -#### ⚠️Successful flight details query check +#### ⚠️ Successful flight details query check **[astm.f3411.v19.NET0710,2](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. -#### ⚠️Flight details data format check +#### ⚠️ Flight details data format check **[astm.f3411.v19.NET0710,2](../../../../requirements/astm/f3411/v19.md)** and **[astm.f3411.v19.NET0340](../../../../requirements/astm/f3411/v19.md)** require a Service Provider to implement the P2P portion of the OpenAPI specification. This check will fail if the response to the flight details endpoint does not validate against the OpenAPI-specified schema. @@ -92,17 +94,17 @@ This implies that any recent position outside the area must be either preceded o In this step, all observers are queried for the flights they observe. Based on the known flights that were injected into the SPs in the first step, these observations are checked against expected behavior/data. Observation rectangles are chosen to encompass the known flights when possible. -#### 🛑Area too large check +#### 🛑 Area too large check **[astm.f3411.v19.NET0430](../../../../requirements/astm/f3411/v19.md)** require that a NetRID Display Provider reject a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. -#### ⚠️Successful observation check +#### ⚠️ Successful observation check Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call to each observer is expected to succeed since a valid view was provided by uss_qualifier. #### [Clustering checks](./display_data_evaluator_clustering.md) -#### ⚠️Duplicate flights check +#### ⚠️ Duplicate flights check Per **[interuss.automated_testing.rid.observation.UniqueFlights](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the same flight ID may not be reported by a Display Provider for two flights in the same observation. @@ -110,11 +112,11 @@ Per **[interuss.automated_testing.rid.observation.UniqueFlights](../../../../req #### [Flight consistency with Common Data Dictionary checks](./common_dictionary_evaluator_dp_flight.md) -#### ⚠️Telemetry being used when present check +#### ⚠️ Telemetry being used when present check **[astm.f3411.v19.NET0290](../../../../requirements/astm/f3411/v19.md)** requires a SP uses Telemetry vs extrapolation when telemetry is present. -#### ⚠️Successful details observation check +#### ⚠️ Successful details observation check Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call for flight details is expected to succeed since a valid ID was provided by uss_qualifier. @@ -122,6 +124,6 @@ Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../. The cleanup phase of this test scenario attempts to remove injected data from all SPs. -### ⚠️Successful test deletion check +### ⚠️ Successful test deletion check **[interuss.automated_testing.rid.injection.DeleteTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)** diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md index e2f8b6e6ab..1e9560464a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md @@ -39,7 +39,7 @@ This test case will: This step attempts to concurrently create multiple ISAs, as specified in this scenario's resource, at the configured DSS. -#### 🛑Concurrent ISAs creation check +#### 🛑 Concurrent ISAs creation check If any of the concurrent ISA creation requests fail or leads to the creation of an incorrect ISA, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. @@ -47,7 +47,7 @@ If any of the concurrent ISA creation requests fail or leads to the creation of This step attempts to concurrently retrieve the previously created ISAs from the DSS. -#### 🛑Successful Concurrent ISA query check +#### 🛑 Successful Concurrent ISA query check If any of the ISAs cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. @@ -55,11 +55,11 @@ If any of the ISAs cannot be queried, the GET ISA DSS endpoint in **[interuss.f3 This test step searches the area in which the ISAs were concurrently created, and expects to find all of them. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑Correct ISAs returned by search check +#### 🛑 Correct ISAs returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. @@ -67,7 +67,7 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts to concurrently delete the earlier created ISAs. -#### 🛑ISAs deletion query success check +#### 🛑 ISAs deletion query success check If an ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. @@ -75,7 +75,7 @@ If an ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030, This step attempts to concurrently access the previously deleted ISAs from the DSS. -#### 🛑ISAs not found check +#### 🛑 ISAs not found check The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -83,11 +83,11 @@ The ISA fetch request was about a deleted ISA, as such the DSS should reject it This step issues a search for active ISAs in the area of the previously deleted ISAs from the DSS. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISAs not returned by search check +#### 🛑 ISAs not returned by search check The ISA search area parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md index 2328796f68..2c823accdd 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md @@ -36,7 +36,7 @@ part of the test. This step attempts to query the configured DSS with the ISA provided as a resource. -#### 🛑ISA created check +#### 🛑 ISA created check If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. @@ -44,13 +44,13 @@ If the ISA cannot be created, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030 This step attempts to retrieve the previously created ISA from the DSS. -#### 🛑Successful ISA query check +#### 🛑 Successful ISA query check If the ISA cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -#### 🛑ISA version match check +#### 🛑 ISA version match check The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after creation, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. @@ -60,7 +60,7 @@ The DSS returns the version of the ISA in the response body. If this version do This step attempts to update the configured DSS with the ISA provided as a resource, with a slightly different end time. -#### 🛑ISA updated check +#### 🛑 ISA updated check If the ISA cannot be updated, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,a](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. @@ -68,13 +68,13 @@ If the ISA cannot be updated, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030 This step attempts to retrieve at the DSS the ISA just updated. -#### 🛑Successful ISA query check +#### 🛑 Successful ISA query check If the ISA cannot be queried, the GET ISA DSS endpoint in **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** is likely not implemented correctly. The DSS returns the ID of the ISA in the response body. If this ID does not match the ID in the resource path, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. -#### 🛑ISA version match check +#### 🛑 ISA version match check The DSS returns the version of the ISA in the response body. If this version does not match the version that was returned after update, and that no modification of the ISA occurred in the meantime, **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** was not implemented correctly and this check will fail. @@ -82,11 +82,11 @@ The DSS returns the version of the ISA in the response body. If this version do This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that overlaps with the resource ISA. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA returned by search check +#### 🛑 ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. @@ -94,11 +94,11 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with the area of the ISA resource and an earliest time that does not overlap with the resource ISA. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA not returned by search check +#### 🛑 ISA not returned by search check The ISA search are parameter cover the resource ISA but the earliest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -106,11 +106,11 @@ The ISA search are parameter cover the resource ISA but the earliest time does n This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that overlaps with the resource ISA. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA returned by search check +#### 🛑 ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -118,11 +118,11 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with the area of the ISA resource and a latest time that does not overlap with the resource ISA. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA not returned by search check +#### 🛑 ISA not returned by search check The ISA search are parameter cover the resource ISA but the latest time does not, as such the resource ISA that exists at the DSS should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -130,11 +130,11 @@ The ISA search are parameter cover the resource ISA but the latest time does not This step attempts an ISA search at the DSS with only the area of the ISA resource. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA returned by search check +#### 🛑 ISA returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -142,7 +142,7 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with an empty search area. -#### 🛑Search request rejected check +#### 🛑 Search request rejected check The search request contained invalid parameters (empty search area), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -150,7 +150,7 @@ The search request contained invalid parameters (empty search area), as such the This step attempts an ISA search at the DSS with a too large search area. -#### 🛑Search request rejected check +#### 🛑 Search request rejected check The search request contained invalid parameters (too large search area), as such the DSS should reject it with a 413 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -158,7 +158,7 @@ The search request contained invalid parameters (too large search area), as such This step attempts an ISA search at the DSS with a polygon defining the area that forms a loop. -#### 🛑Search request rejected check +#### 🛑 Search request rejected check The search request contained invalid parameters (area polygon is a loop, which is not allowed), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -168,7 +168,7 @@ The search request contained invalid parameters (area polygon is a loop, which i This step attempts an ISA deletion with a wrong version. -#### 🛑Delete request rejected check +#### 🛑 Delete request rejected check The deletion request contained invalid parameters (wrong version), as such the DSS should reject it with a 409 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)**. @@ -176,7 +176,7 @@ The deletion request contained invalid parameters (wrong version), as such the D This step attempts an ISA deletion with an empty version. -#### 🛑Delete request rejected check +#### 🛑 Delete request rejected check The deletion request contained invalid parameters (empty version), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)**. @@ -184,7 +184,7 @@ The deletion request contained invalid parameters (empty version), as such the D This step attempts an ISA deletion at the DSS. -#### 🛑ISA deleted check +#### 🛑 ISA deleted check If the ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** is likely not implemented correctly. @@ -192,7 +192,7 @@ If the ISA cannot be deleted, the PUT DSS endpoint in **[astm.f3411.v22a.DSS0030 This step attempts to retrieve at the DSS the ISA just deleted. -#### 🛑ISA not found check +#### 🛑 ISA not found check The ISA fetch request was about a deleted ISA, as such the DSS should reject it with a 404 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -200,11 +200,11 @@ The ISA fetch request was about a deleted ISA, as such the DSS should reject it This step attempts an ISA search at the DSS with only the area of the ISA resource. Since it has just been deleted, the ISA should not be returned. -#### 🛑Successful ISAs search check +#### 🛑 Successful ISAs search check The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑ISA not returned by search check +#### 🛑 ISA not returned by search check The ISA search are parameter cover the resource ISA, but it has been previously deleted, as such the ISA should not be returned by the search. If it is returned, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -212,14 +212,14 @@ The ISA search are parameter cover the resource ISA, but it has been previously The cleanup phase of this test scenario attempts to remove the ISA if the test ended prematurely. -### 🛑Successful ISA query check +### 🛑 Successful ISA query check **[interuss.f3411.dss_endpoints.GetISA](../../../../../requirements/interuss/f3411/dss_endpoints.md)** requires the implementation of the DSS endpoint enabling retrieval of information about a specific ISA; if the individual ISA cannot be retrieved and the error isn't a 404, then this requirement isn't met. -### 🛑Removed pre-existing ISA check +### 🛑 Removed pre-existing ISA check If an ISA with the intended ID is still present in the DSS, it needs to be removed before exiting the test. If that ISA cannot be deleted, then the **[astm.f3411.v22a.DSS0030,b](../../../../../requirements/astm/f3411/v22a.md)** requirement to implement the ISA deletion endpoint might not be met. -### ⚠️Notified subscriber check +### ⚠️ Notified subscriber check When an ISA is deleted, subscribers must be notified. If a subscriber cannot be notified, that subscriber USS did not correctly implement "POST Identification Service Area" in **[astm.f3411.v22a.NET0730](../../../../../requirements/astm/f3411/v22a.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md index ef94e45df1..08a7a81ea8 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/test_steps/put_isa.md @@ -3,7 +3,7 @@ This page describes the content of a common test step where a creation or an update of an ISA should be successful. See `DSSWrapper.put_isa` in [`dss_wrapper.py`](../../../dss_wrapper.py). -## ℹ️ISA response code check +## ℹ️ ISA response code check The API for **[astm.f3411.v22a.DSS0030,a](../../../../../../requirements/astm/f3411/v22a.md)** specifies that the code of successful DSS responses is 200. While tolerated in some cases, if the DSS responds with an HTTP code 201 for success, this check will fail with a low severity. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md index 46144925b0..e8afd32488 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/misbehavior.md @@ -28,14 +28,14 @@ A [`DSSInstanceResource`](../../../../resources/astm/f3411/dss.py) is required f In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. -#### 🛑Successful injection check +#### 🛑 Successful injection check Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. **[astm.f3411.v22a.NET0500](../../../../requirements/astm/f3411/v22a.md)** requires a Service Provider to provide a persistently supported test instance of their implementation. This check will fail if the flight was not successfully injected. -#### 🛑Identifiable flights check +#### 🛑 Identifiable flights check This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md index 4883b80654..ed1944b936 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md @@ -32,7 +32,7 @@ If specified, uss_qualifier will act as a Display Provider and check a DSS insta In this step, uss_qualifier injects a single nominal flight into each SP under test, usually with a start time in the future. Each SP is expected to queue the provided telemetry and later simulate that telemetry coming from an aircraft at the designated timestamps. -#### 🛑Successful injection check +#### 🛑 Successful injection check Per **[interuss.automated_testing.rid.injection.UpsertTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the injection attempt of the valid flight should succeed for every NetRID Service Provider under test. @@ -46,7 +46,7 @@ TODO: Validate injected flights, especially to make sure they contain the specif Per **[interuss.automated_testing.rid.injection.UpsertTestResult](../../../../requirements/interuss/automated_testing/rid/injection.md)**, the NetRID Service Provider under test should only make valid modifications to the injected flights. This includes: * A flight with the specified injection ID must be returned. -#### 🛑Identifiable flights check +#### 🛑 Identifiable flights check This particular test requires each flight to be uniquely identifiable by its 2D telemetry position; the same (lat, lng) pair may not appear in two different telemetry points, even if the two points are in different injected flights. This should generally be achieved by injecting appropriate data. @@ -54,37 +54,39 @@ This particular test requires each flight to be uniquely identifiable by its 2D If a DSS was provided to this test scenario, uss_qualifier acts as a Display Provider to query Service Providers under test in this step. -#### ⚠️ISA query check +#### ⚠️ ISA query check **[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. If uss_qualifier is unable to query the DSS for ISAs, this check will fail. -#### 🛑Area too large check +#### 🛑 Area too large check **[astm.f3411.v22a.NET0250](../../../../requirements/astm/f3411/v22a.md)** requires that a NetRID Service Provider rejects a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. #### [Flight presence checks](./display_data_evaluator_flight_presence.md) -#### ⚠️Flights data format check +#### ⚠️ Flights data format check **[astm.f3411.v22a.NET0710,1](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md)** requires a Service Provider to implement the P2P portion of the OpenAPI specification. This check will fail if the response to the /flights endpoint does not validate against the OpenAPI-specified schema. #### [Flight consistency with Common Data Dictionary checks](./common_dictionary_evaluator_sp_flight.md) -#### ⚠️Recent positions timestamps check +#### ⚠️ Recent positions timestamps check + **[astm.f3411.v22a.NET0270](../../../../requirements/astm/f3411/v22a.md)** requires all recent positions to be within the NetMaxNearRealTimeDataPeriod. This check will fail if any of the reported positions are older than the maximally allowed period plus NetSpDataResponseTime99thPercentile. -#### ⚠️Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing check +#### ⚠️ Recent positions for aircraft crossing the requested area boundary show only one position before or after crossing check + **[astm.f3411.v22a.NET0270](../../../../requirements/astm/f3411/v22a.md)** requires that when an aircraft enters or leaves the queried area, the last or first reported position outside the area is provided in the recent positions, as long as it is not older than NetMaxNearRealTimeDataPeriod. This implies that any recent position outside the area must be either preceded or followed by a position inside the area. (This check validates NET0270 b and c). -#### ⚠️Successful flight details query check +#### ⚠️ Successful flight details query check **[astm.f3411.v22a.NET0710,2](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the GET flight details endpoint. This check will fail if uss_qualifier cannot query that endpoint (specified in the ISA present in the DSS) successfully. -#### ⚠️Flight details data format check +#### ⚠️ Flight details data format check **[astm.f3411.v22a.NET0710,2](../../../../requirements/astm/f3411/v22a.md)** and **[astm.f3411.v22a.NET0340](../../../../requirements/astm/f3411/v22a.md) require a Service Provider to implement the P2P portion of the OpenAPI specification. This check will fail if the response to the flight details endpoint does not validate against the OpenAPI-specified schema. @@ -94,17 +96,17 @@ This implies that any recent position outside the area must be either preceded o In this step, all observers are queried for the flights they observe. Based on the known flights that were injected into the SPs in the first step, these observations are checked against expected behavior/data. Observation rectangles are chosen to encompass the known flights when possible. -#### 🛑Area too large check +#### 🛑 Area too large check **[astm.f3411.v22a.NET0430](../../../../requirements/astm/f3411/v22a.md)** require that a NetRID Display Provider reject a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. -#### ⚠️Successful observation check +#### ⚠️ Successful observation check Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call to each observer is expected to succeed since a valid view was provided by uss_qualifier. #### [Clustering checks](./display_data_evaluator_clustering.md) -#### ⚠️Duplicate flights check +#### ⚠️ Duplicate flights check Per **[interuss.automated_testing.rid.observation.UniqueFlights](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the same flight ID may not be reported by a Display Provider for two flights in the same observation. @@ -112,11 +114,11 @@ Per **[interuss.automated_testing.rid.observation.UniqueFlights](../../../../req #### [Flight consistency with Common Data Dictionary checks](./common_dictionary_evaluator_dp_flight.md) -#### ⚠️Telemetry being used when present check +#### ⚠️ Telemetry being used when present check **[astm.f3411.v22a.NET0290](../../../../requirements/astm/f3411/v22a.md)** requires a SP uses Telemetry vs extrapolation when telemetry is present. -#### ⚠️Successful details observation check +#### ⚠️ Successful details observation check Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../../requirements/interuss/automated_testing/rid/observation.md)**, the call for flight details is expected to succeed since a valid ID was provided by uss_qualifier. @@ -126,6 +128,6 @@ Per **[interuss.automated_testing.rid.observation.ObservationSuccess](../../../. The cleanup phase of this test scenario attempts to remove injected data from all SPs. -### ⚠️Successful test deletion check +### ⚠️ Successful test deletion check **[interuss.automated_testing.rid.injection.DeleteTestSuccess](../../../../requirements/interuss/automated_testing/rid/injection.md)** From 93af5202b105d9f8c885b5d1f3a5f36096f28486 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Wed, 15 Jan 2025 16:21:45 +0100 Subject: [PATCH 7/8] Corrections from PR --- .../astm/netrid/v19/dss/heavy_traffic_concurrent.md | 2 +- .../scenarios/astm/netrid/v19/dss/isa_simple.md | 6 +++--- .../scenarios/astm/netrid/v19/nominal_behavior.md | 4 ++-- .../astm/netrid/v22a/dss/heavy_traffic_concurrent.md | 2 +- .../scenarios/astm/netrid/v22a/dss/isa_simple.md | 6 +++--- .../scenarios/astm/netrid/v22a/nominal_behavior.md | 4 ++-- monitoring/uss_qualifier/scenarios/scenario.py | 6 ++++++ 7 files changed, 18 insertions(+), 12 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md index 6b6d812187..34b362d654 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/heavy_traffic_concurrent.md @@ -59,7 +59,7 @@ This test step searches the area in which the ISAs were concurrently created, an The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑 Correct ISAs returned by search check +#### ⚠️ Correct ISAs returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md index 1cce0f2f95..d0069fc063 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/dss/isa_simple.md @@ -142,7 +142,7 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with an empty search area. -#### 🛑 Search request rejected check +#### ⚠️ Search request rejected check The search request contained invalid parameters (empty search area), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -150,7 +150,7 @@ The search request contained invalid parameters (empty search area), as such the This step attempts an ISA search at the DSS with a too large search area. -#### 🛑 Search request rejected check +#### ⚠️ Search request rejected check The search request contained invalid parameters (too large search area), as such the DSS should reject it with a 413 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -158,7 +158,7 @@ The search request contained invalid parameters (too large search area), as such This step attempts an ISA search at the DSS with a polygon defining the area that forms a loop. -#### 🛑 Search request rejected check +#### ⚠️ Search request rejected check The search request contained invalid parameters (area polygon is a loop, which is not allowed), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md index c23f74046b..5cf14bec10 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v19/nominal_behavior.md @@ -58,7 +58,7 @@ If a DSS was provided to this test scenario, uss_qualifier acts as a Display Pro **[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. If uss_qualifier is unable to query the DSS for ISAs, this check will fail. -#### 🛑 Area too large check +#### ⚠️ Area too large check **[astm.f3411.v19.NET0250](../../../../requirements/astm/f3411/v19.md)** requires that a NetRID Service Provider rejects a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. @@ -94,7 +94,7 @@ This implies that any recent position outside the area must be either preceded o In this step, all observers are queried for the flights they observe. Based on the known flights that were injected into the SPs in the first step, these observations are checked against expected behavior/data. Observation rectangles are chosen to encompass the known flights when possible. -#### 🛑 Area too large check +#### ⚠️ Area too large check **[astm.f3411.v19.NET0430](../../../../requirements/astm/f3411/v19.md)** require that a NetRID Display Provider reject a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md index 1e9560464a..045c55639c 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/heavy_traffic_concurrent.md @@ -59,7 +59,7 @@ This test step searches the area in which the ISAs were concurrently created, an The ISA search parameters are valid, as such the search should be successful. If the request is not successful, this check will fail per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. -#### 🛑 Correct ISAs returned by search check +#### ⚠️ Correct ISAs returned by search check The ISA search parameters cover the resource ISA, as such the resource ISA that exists at the DSS should be returned by the search. If it is not returned, this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md index 2c823accdd..a59a64255a 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/dss/isa_simple.md @@ -142,7 +142,7 @@ The ISA search parameters cover the resource ISA, as such the resource ISA that This step attempts an ISA search at the DSS with an empty search area. -#### 🛑 Search request rejected check +#### ⚠️ Search request rejected check The search request contained invalid parameters (empty search area), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -150,7 +150,7 @@ The search request contained invalid parameters (empty search area), as such the This step attempts an ISA search at the DSS with a too large search area. -#### 🛑 Search request rejected check +#### ⚠️ Search request rejected check The search request contained invalid parameters (too large search area), as such the DSS should reject it with a 413 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. @@ -158,7 +158,7 @@ The search request contained invalid parameters (too large search area), as such This step attempts an ISA search at the DSS with a polygon defining the area that forms a loop. -#### 🛑 Search request rejected check +#### ⚠️ Search request rejected check The search request contained invalid parameters (area polygon is a loop, which is not allowed), as such the DSS should reject it with a 400 HTTP code. If the DSS responds successfully to this request, or if it rejected with an incorrect HTTP code, this check will fail as per **[interuss.f3411.dss_endpoints.SearchISAs](../../../../../requirements/interuss/f3411/dss_endpoints.md)**. diff --git a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md index ed1944b936..068a27e2ec 100644 --- a/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md +++ b/monitoring/uss_qualifier/scenarios/astm/netrid/v22a/nominal_behavior.md @@ -58,7 +58,7 @@ If a DSS was provided to this test scenario, uss_qualifier acts as a Display Pro **[interuss.f3411.dss_endpoints.SearchISAs](../../../../requirements/interuss/f3411/dss_endpoints.md)** requires a USS providing a DSS instance to implement the DSS endpoints of the OpenAPI specification. If uss_qualifier is unable to query the DSS for ISAs, this check will fail. -#### 🛑 Area too large check +#### ⚠️ Area too large check **[astm.f3411.v22a.NET0250](../../../../requirements/astm/f3411/v22a.md)** requires that a NetRID Service Provider rejects a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. @@ -96,7 +96,7 @@ This implies that any recent position outside the area must be either preceded o In this step, all observers are queried for the flights they observe. Based on the known flights that were injected into the SPs in the first step, these observations are checked against expected behavior/data. Observation rectangles are chosen to encompass the known flights when possible. -#### 🛑 Area too large check +#### ⚠️ Area too large check **[astm.f3411.v22a.NET0430](../../../../requirements/astm/f3411/v22a.md)** require that a NetRID Display Provider reject a request for a very large view area with a diagonal greater than *NetMaxDisplayAreaDiagonal*. If such a large view is requested and a 413 error code is not received, then this check will fail. diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index dff0b98429..91e8276fe9 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -453,6 +453,12 @@ def check( else: check_list = ", ".join(available_checks) if self._allow_undocumented_checks: + # We create a dunny TestCheckDocumentation to continue. + # The severity is unknown since we don't have documentation, + # but we default to a medium severity (so a failure won't stop + # tests). + # Thoses undocumented checks are primary used for unit testing, + # this shouldn't have any impact on normal testing. check_documentation = TestCheckDocumentation( name=name, applicable_requirements=[], From b033be979428974cb10444732cd89ddc13297a03 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Thu, 16 Jan 2025 09:15:57 +0100 Subject: [PATCH 8/8] Typos --- monitoring/uss_qualifier/scenarios/scenario.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/uss_qualifier/scenarios/scenario.py b/monitoring/uss_qualifier/scenarios/scenario.py index 91e8276fe9..bce5f4d5dc 100644 --- a/monitoring/uss_qualifier/scenarios/scenario.py +++ b/monitoring/uss_qualifier/scenarios/scenario.py @@ -453,11 +453,11 @@ def check( else: check_list = ", ".join(available_checks) if self._allow_undocumented_checks: - # We create a dunny TestCheckDocumentation to continue. + # We create a dummy TestCheckDocumentation to continue. # The severity is unknown since we don't have documentation, # but we default to a medium severity (so a failure won't stop # tests). - # Thoses undocumented checks are primary used for unit testing, + # These undocumented checks are primary used for unit testing, # this shouldn't have any impact on normal testing. check_documentation = TestCheckDocumentation( name=name,