From dfb92aef3c6a918907faa4fce4d083a2561b3cdc Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Fri, 1 Jul 2022 12:32:19 +0300 Subject: [PATCH 1/6] Issues #562,#561,#560: - No source of entropy in _get_nearest_mirrors_by_network_data() - _get_nearest_mirrors_by_network_data() fails to exclude near-by private mirrors for extra options. - Exclude the private mirrors from the mirrors list in the case of fallback behavior --- src/backend/api/handlers.py | 40 +++++++++++++++++++++++++++++-------- src/backend/api/utils.py | 3 ++- src/backend/yaml_snippets | 2 +- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/backend/api/handlers.py b/src/backend/api/handlers.py index afd54d5f..40b30037 100644 --- a/src/backend/api/handlers.py +++ b/src/backend/api/handlers.py @@ -78,6 +78,21 @@ async def _get_nearest_mirrors_by_network_data( The function returns mirrors which are in the same subnet or have the same ASN as a request's IP """ + + def _is_additional_mirrors_suitable( + mirror_data: MirrorData, + main_list_of_mirrors: list[MirrorData] + ) -> bool: + """ + An additional mirror is a mirror + which is fresh (not outdated), not flapping and public, because + all suitable private mirrors we already found, + using ASN or subnets data + """ + return mirror_data.status == 'ok' and \ + not mirror_data.private and \ + mirror_data not in main_list_of_mirrors + match = get_geo_data_by_ip(ip_address) asn = get_asn_by_ip(ip_address) suitable_mirrors = [] @@ -105,14 +120,23 @@ async def _get_nearest_mirrors_by_network_data( if 1 <= len(suitable_mirrors) < LENGTH_CLOUD_MIRRORS_LIST\ and match is not None: continent, country, _, _, latitude, longitude = match + not_sorted_additional_mirrors = [ + mirror for mirror in mirrors if _is_additional_mirrors_suitable( + mirror_data=mirror, + main_list_of_mirrors=suitable_mirrors, + ) + ] + sorted_additional_mirrors = sort_mirrors_by_distance_and_country( + request_geo_data=(latitude, longitude), + mirrors=not_sorted_additional_mirrors, + country=country, + ) + randomized_additional_mirrors = randomize_mirrors_within_distance( + mirrors=sorted_additional_mirrors, + country=country, + )[:LENGTH_CLOUD_MIRRORS_LIST - len(suitable_mirrors)] suitable_mirrors.extend( - mirror['mirror'] for mirror in - sort_mirrors_by_distance_and_country( - request_geo_data=(latitude, longitude), - mirrors=[mirror for mirror in mirrors - if mirror not in suitable_mirrors], - country=country, - )[:LENGTH_CLOUD_MIRRORS_LIST - len(suitable_mirrors)] + mirror['mirror'] for mirror in randomized_additional_mirrors ) return suitable_mirrors @@ -186,7 +210,7 @@ async def _get_nearest_mirrors( if not suitable_mirrors: suitable_mirrors = await _get_nearest_mirrors_by_geo_data( ip_address=ip_address, - without_private_mirrors=without_private_mirrors, + without_private_mirrors=True, ) await set_mirrors_to_cache( ip_address, diff --git a/src/backend/api/utils.py b/src/backend/api/utils.py index ee6bf24e..0e87b7a8 100644 --- a/src/backend/api/utils.py +++ b/src/backend/api/utils.py @@ -152,7 +152,8 @@ def get_geo_data_by_ip( db = GeoIPEngine.get_instance() try: city = db.city(ip) - except AddressNotFoundError: + # ValueError will be raised in case of incorrect IP + except (AddressNotFoundError, ValueError): return try: city_name = city.city.name diff --git a/src/backend/yaml_snippets b/src/backend/yaml_snippets index 76213896..1dd0a625 160000 --- a/src/backend/yaml_snippets +++ b/src/backend/yaml_snippets @@ -1 +1 @@ -Subproject commit 76213896d0cae1911e1c869a524d86a916bdc804 +Subproject commit 1dd0a62524ff0404fba41e3efe1064c44081771d From b715adbfdae3dfff5fe873af6d3c0df1f8f60cbb Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Fri, 1 Jul 2022 12:41:45 +0300 Subject: [PATCH 2/6] Issues #562,#561,#560: - No source of entropy in _get_nearest_mirrors_by_network_data() - _get_nearest_mirrors_by_network_data() fails to exclude near-by private mirrors for extra options. - Exclude the private mirrors from the mirrors list in the case of fallback behavior The Azure mirrors have allowed list of arches --- src/backend/api/handlers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/backend/api/handlers.py b/src/backend/api/handlers.py index 40b30037..04978c5c 100644 --- a/src/backend/api/handlers.py +++ b/src/backend/api/handlers.py @@ -135,9 +135,7 @@ def _is_additional_mirrors_suitable( mirrors=sorted_additional_mirrors, country=country, )[:LENGTH_CLOUD_MIRRORS_LIST - len(suitable_mirrors)] - suitable_mirrors.extend( - mirror['mirror'] for mirror in randomized_additional_mirrors - ) + suitable_mirrors.extend(randomized_additional_mirrors) return suitable_mirrors From c56893ce7b86e6e67be93d824d84edad685de377 Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Fri, 1 Jul 2022 13:11:44 +0300 Subject: [PATCH 3/6] Issues #562,#561,#560: - No source of entropy in _get_nearest_mirrors_by_network_data() - _get_nearest_mirrors_by_network_data() fails to exclude near-by private mirrors for extra options. - Exclude the private mirrors from the mirrors list in the case of fallback behavior - The Azure mirrors have allowed list of arches - Decrease level of logging messages in some cases - Cache subnets of Azure/AWS cloud --- src/backend/api/mirrors_update.py | 4 ++-- src/backend/api/redis.py | 31 +++++++++++++++++++++++++++++++ src/backend/api/utils.py | 14 ++++++++++++-- src/backend/yaml_snippets | 2 +- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/backend/api/mirrors_update.py b/src/backend/api/mirrors_update.py index cefb67bc..a7525a86 100644 --- a/src/backend/api/mirrors_update.py +++ b/src/backend/api/mirrors_update.py @@ -93,7 +93,7 @@ async def set_repo_status( ) as resp: timestamp_response = await resp.text() except (asyncio.exceptions.TimeoutError, HTTPError): - logger.error( + logger.warning( 'Mirror "%s" has no timestamp file by url "%s"', mirror_info.name, timestamp_url, @@ -107,7 +107,7 @@ async def set_repo_status( try: mirror_last_updated = float(timestamp_response) except ValueError: - logger.info( + logger.warning( 'Mirror "%s" has broken timestamp file by url "%s"', mirror_info.name, timestamp_url, diff --git a/src/backend/api/redis.py b/src/backend/api/redis.py index bf18ab08..9fdd96e8 100644 --- a/src/backend/api/redis.py +++ b/src/backend/api/redis.py @@ -66,6 +66,37 @@ async def set_mirrors_to_cache( ) +async def get_subnets_from_cache( + key: str, +) -> dict: + """ + Get a cached subnets of Azure/AWS cloud + """ + async with redis_context() as redis_engine: + subnets_string = await redis_engine.get(str(key)) + if subnets_string is not None: + subnets_json = json.loads( + subnets_string, + ) + return subnets_json + + +async def set_subnets_to_cache( + key: str, + subnets: dict, +) -> None: + """ + Save a mirror list for specified IP to cache + """ + async with redis_context() as redis_engine: + subnets = json.dumps(subnets) + await redis_engine.set( + str(key), + subnets, + 24 * 60 * 60, + ) + + async def get_geolocation_from_cache( key: str ) -> Union[tuple[float, float], tuple[None, None]]: diff --git a/src/backend/api/utils.py b/src/backend/api/utils.py index 0e87b7a8..36220812 100644 --- a/src/backend/api/utils.py +++ b/src/backend/api/utils.py @@ -43,7 +43,9 @@ from haversine import haversine from api.redis import ( get_geolocation_from_cache, - set_geolocation_to_cache + set_geolocation_to_cache, + get_subnets_from_cache, + set_subnets_to_cache, ) logger = get_logger(__name__) @@ -210,7 +212,7 @@ async def get_azure_subnets_json(http_session: ClientSession) -> Optional[dict]: response_json = await resp.json( content_type='application/octet-stream', ) - except (ClientConnectorError, TimeoutError) as err: + except (ClientConnectorError, asyncio.exceptions.TimeoutError) as err: logger.error( 'Cannot get json with Azure subnets by url "%s" because "%s"', link_to_json_url, @@ -240,6 +242,9 @@ async def get_aws_subnets_json(http_session: ClientSession) -> Optional[dict]: async def get_azure_subnets(http_session: ClientSession): + subnets = await get_subnets_from_cache('azure_subnets') + if subnets is not None: + return subnets data_json = await get_azure_subnets_json(http_session=http_session) subnets = dict() if data_json is None: @@ -250,10 +255,14 @@ async def get_azure_subnets(http_session: ClientSession): properties = value['properties'] subnets[properties['region'].lower()] = \ properties['addressPrefixes'] + await set_subnets_to_cache('aws_subnets', subnets) return subnets async def get_aws_subnets(http_session: ClientSession): + subnets = await get_subnets_from_cache('aws_subnets') + if subnets is not None: + return subnets data_json = await get_aws_subnets_json(http_session=http_session) subnets = defaultdict(list) if data_json is None: @@ -262,6 +271,7 @@ async def get_aws_subnets(http_session: ClientSession): subnets[v4_prefix['region'].lower()].append(v4_prefix['ip_prefix']) for v6_prefix in data_json['ipv6_prefixes']: subnets[v6_prefix['region'].lower()].append(v6_prefix['ipv6_prefix']) + await set_subnets_to_cache('aws_subnets', subnets) return subnets diff --git a/src/backend/yaml_snippets b/src/backend/yaml_snippets index 1dd0a625..1f9f85f1 160000 --- a/src/backend/yaml_snippets +++ b/src/backend/yaml_snippets @@ -1 +1 @@ -Subproject commit 1dd0a62524ff0404fba41e3efe1064c44081771d +Subproject commit 1f9f85f16a50f82241be3fad49e20b7c1f65f69d From 032b0086e5e2f2ab1aa4533ad8c131ab75635150 Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Fri, 1 Jul 2022 13:46:06 +0300 Subject: [PATCH 4/6] Issues #562,#561,#560: - No source of entropy in _get_nearest_mirrors_by_network_data() - _get_nearest_mirrors_by_network_data() fails to exclude near-by private mirrors for extra options. - Exclude the private mirrors from the mirrors list in the case of fallback behavior - The Azure mirrors have allowed list of arches - Decrease level of logging messages in some cases - Cache subnets of Azure/AWS cloud --- src/backend/yaml_snippets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/yaml_snippets b/src/backend/yaml_snippets index 1f9f85f1..3a17b0fb 160000 --- a/src/backend/yaml_snippets +++ b/src/backend/yaml_snippets @@ -1 +1 @@ -Subproject commit 1f9f85f16a50f82241be3fad49e20b7c1f65f69d +Subproject commit 3a17b0fb909e6dfc47aea0bac420592b49c42af0 From 1471030737dc043dfa730e1a75b4cdf6a4a2b837 Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Fri, 1 Jul 2022 13:48:10 +0300 Subject: [PATCH 5/6] Issues #562,#561,#560: - No source of entropy in _get_nearest_mirrors_by_network_data() - _get_nearest_mirrors_by_network_data() fails to exclude near-by private mirrors for extra options. - Exclude the private mirrors from the mirrors list in the case of fallback behavior - The Azure mirrors have allowed list of arches - Decrease level of logging messages in some cases - Cache subnets of Azure/AWS cloud --- src/backend/yaml_snippets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/yaml_snippets b/src/backend/yaml_snippets index 3a17b0fb..b813bd50 160000 --- a/src/backend/yaml_snippets +++ b/src/backend/yaml_snippets @@ -1 +1 @@ -Subproject commit 3a17b0fb909e6dfc47aea0bac420592b49c42af0 +Subproject commit b813bd50abd5711a41dfbd000cc0965ffbc5640e From 6802d4b831348882874e9c2c70a1fffc245a0f23 Mon Sep 17 00:00:00 2001 From: soksanichenko Date: Mon, 4 Jul 2022 13:15:39 +0300 Subject: [PATCH 6/6] Issues #562,#561,#560: - No source of entropy in _get_nearest_mirrors_by_network_data() - _get_nearest_mirrors_by_network_data() fails to exclude near-by private mirrors for extra options. - Exclude the private mirrors from the mirrors list in the case of fallback behavior - The Azure mirrors have allowed list of arches - Decrease level of logging messages in some cases - Cache subnets of Azure/AWS cloud --- src/backend/api/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/api/utils.py b/src/backend/api/utils.py index 36220812..b4ee6c77 100644 --- a/src/backend/api/utils.py +++ b/src/backend/api/utils.py @@ -255,7 +255,7 @@ async def get_azure_subnets(http_session: ClientSession): properties = value['properties'] subnets[properties['region'].lower()] = \ properties['addressPrefixes'] - await set_subnets_to_cache('aws_subnets', subnets) + await set_subnets_to_cache('azure_subnets', subnets) return subnets