diff --git a/.isort.cfg b/.isort.cfg index b90ca30c..7734b47c 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,2 +1,2 @@ [settings] -known_third_party = cf_units,cftime,httpretty,isodate,lxml,netCDF4,numpy,owslib,pendulum,pkg_resources,pygeoif,pyproj,pytest,regex,requests,setuptools,validators +known_third_party = cf_units,cftime,distutils,httpretty,isodate,lxml,netCDF4,numpy,owslib,pendulum,pkg_resources,pygeoif,pyproj,pytest,regex,requests,requests_mock,setuptools,shapely,validators diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d60f6b78..0b2bd1f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,27 +11,20 @@ repos: - id: check-docstring-first - id: check-added-large-files - id: requirements-txt-fixer - - id: file-contents-sorter - files: test_requirements.txt -- repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 +- repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 hooks: - id: flake8 exclude: docs/source/conf.py args: [--max-line-length=200, "--ignore=E203,E501,W503", "--select=select=C,E,F,W,B,B950"] -- repo: https://github.com/pre-commit/mirrors-isort - rev: v5.10.1 +- repo: https://github.com/pycqa/isort + rev: 5.10.1 hooks: - id: isort additional_dependencies: [toml] - args: [--project=compliance_checker, --multi-line=3, --lines-after-imports=2, --lines-between-types=1, --trailing-comma, --force-grid-wrap=0, --use-parentheses, --line-width=88] - -- repo: https://github.com/asottile/seed-isort-config - rev: v2.2.0 - hooks: - - id: seed-isort-config + args: ["--profile", "black", "--filter-files"] - repo: https://github.com/psf/black rev: 22.3.0 diff --git a/cchecker.py b/cchecker.py index 243fc737..4155f057 100755 --- a/cchecker.py +++ b/cchecker.py @@ -3,7 +3,6 @@ import argparse import sys import warnings - from collections import defaultdict from textwrap import dedent diff --git a/compliance_checker/__init__.py b/compliance_checker/__init__.py index a51d505e..1a25bdef 100644 --- a/compliance_checker/__init__.py +++ b/compliance_checker/__init__.py @@ -5,7 +5,6 @@ from netCDF4 import Dataset - try: from ._version import __version__ except ImportError: diff --git a/compliance_checker/acdd.py b/compliance_checker/acdd.py index 7f490df5..40bcfbd1 100644 --- a/compliance_checker/acdd.py +++ b/compliance_checker/acdd.py @@ -10,7 +10,6 @@ import numpy as np import pendulum - from cftime import num2pydate from pygeoif import from_wkt @@ -582,7 +581,7 @@ def check_time_extents(self, ds): try: t_min = dateparse(ds.time_coverage_start) t_max = dateparse(ds.time_coverage_end) - except: + except (TypeError, pendulum.parsing.exceptions.ParserError): return Result( BaseCheck.MEDIUM, False, @@ -618,7 +617,7 @@ def check_time_extents(self, ds): num2pydate(ds.variables[timevar][-1], ds.variables[timevar].units), "UTC", ) - except: + except ValueError: return Result( BaseCheck.MEDIUM, False, @@ -829,7 +828,8 @@ def check_var_coverage_content_type(self, ds): } if ctype not in valid_ctypes: msgs.append( - 'coverage_content_type "%s" not in %s' % (variable, sorted(valid_ctypes)) + 'coverage_content_type "%s" not in %s' + % (variable, sorted(valid_ctypes)) ) results.append( Result( diff --git a/compliance_checker/base.py b/compliance_checker/base.py index 6bebcd8a..acd080bd 100644 --- a/compliance_checker/base.py +++ b/compliance_checker/base.py @@ -8,13 +8,11 @@ import pprint import re import warnings - from collections import defaultdict from functools import wraps from io import StringIO import validators - from lxml import etree from netCDF4 import Dataset from owslib.namespaces import Namespaces @@ -22,11 +20,9 @@ from owslib.swe.sensor.sml import SensorML import compliance_checker.cfutil as cfutil - from compliance_checker import MemoizedDataset, __version__ from compliance_checker.util import kvp_convert - # Python 3.5+ should work, also have a fallback try: from typing import Pattern diff --git a/compliance_checker/cf/__init__.py b/compliance_checker/cf/__init__.py index 573f9508..1baa8d5d 100644 --- a/compliance_checker/cf/__init__.py +++ b/compliance_checker/cf/__init__.py @@ -1,10 +1,9 @@ +from compliance_checker.cf import util from compliance_checker.cf.appendix_d import ( dimless_vertical_coordinates_1_6, dimless_vertical_coordinates_1_7, ) from compliance_checker.cf.cf import CF1_6Check, CF1_7Check -from compliance_checker.cf import util - __all__ = [ "CF1_6Check", diff --git a/compliance_checker/cf/cf.py b/compliance_checker/cf/cf.py index 747e3b23..41b8d8ab 100644 --- a/compliance_checker/cf/cf.py +++ b/compliance_checker/cf/cf.py @@ -2,17 +2,22 @@ # -*- coding: utf-8 -*- from compliance_checker import cfutil # noqa: F401 -from compliance_checker.base import BaseCheck, BaseNCCheck, Result, TestCtx # noqa: F401 +from compliance_checker.base import ( # noqa: F401 + BaseCheck, + BaseNCCheck, + Result, + TestCtx, +) from compliance_checker.cf import util # noqa: F401 from compliance_checker.cf.appendix_d import ( # noqa: F401 dimless_vertical_coordinates_1_6, dimless_vertical_coordinates_1_7, no_missing_terms, ) - -from compliance_checker.cf.appendix_e import cell_methods16, cell_methods17 # noqa: F401 +from compliance_checker.cf.appendix_e import cell_methods16 # noqa: F401 +from compliance_checker.cf.appendix_e import cell_methods17 # noqa: F401 +from compliance_checker.cf.appendix_f import ellipsoid_names17 # noqa: F401 from compliance_checker.cf.appendix_f import ( # noqa: F401 - ellipsoid_names17, grid_mapping_attr_types16, grid_mapping_attr_types17, grid_mapping_dict16, diff --git a/compliance_checker/cf/cf_1_6.py b/compliance_checker/cf/cf_1_6.py index fb87b1d6..a4006b09 100644 --- a/compliance_checker/cf/cf_1_6.py +++ b/compliance_checker/cf/cf_1_6.py @@ -1,27 +1,22 @@ +import difflib import logging - from collections import defaultdict import numpy as np import regex - from cf_units import Unit from compliance_checker import cfutil from compliance_checker.base import BaseCheck, Result, TestCtx from compliance_checker.cf import util from compliance_checker.cf.appendix_c import valid_modifiers -from compliance_checker.cf.appendix_d import ( - dimless_vertical_coordinates_1_6, -) +from compliance_checker.cf.appendix_d import dimless_vertical_coordinates_1_6 from compliance_checker.cf.appendix_e import cell_methods16 from compliance_checker.cf.appendix_f import ( grid_mapping_attr_types16, grid_mapping_dict16, ) - from compliance_checker.cf.cf_base import CFNCCheck, appendix_a_base -import difflib logger = logging.getLogger(__name__) diff --git a/compliance_checker/cf/cf_1_7.py b/compliance_checker/cf/cf_1_7.py index c4728189..0729dacb 100644 --- a/compliance_checker/cf/cf_1_7.py +++ b/compliance_checker/cf/cf_1_7.py @@ -1,19 +1,15 @@ import logging import os import sqlite3 - from warnings import warn import numpy as np import pyproj import regex - from compliance_checker import cfutil from compliance_checker.base import BaseCheck, Result, TestCtx -from compliance_checker.cf.appendix_d import ( - dimless_vertical_coordinates_1_7, -) +from compliance_checker.cf.appendix_d import dimless_vertical_coordinates_1_7 from compliance_checker.cf.appendix_e import cell_methods17 from compliance_checker.cf.appendix_f import ( ellipsoid_names17, @@ -22,8 +18,8 @@ horizontal_datum_names17, prime_meridian_names17, ) -from compliance_checker.cf.cf_base import appendix_a_base from compliance_checker.cf.cf_1_6 import CF1_6Check +from compliance_checker.cf.cf_base import appendix_a_base logger = logging.getLogger(__name__) diff --git a/compliance_checker/cf/cf_1_8.py b/compliance_checker/cf/cf_1_8.py index f2a15548..fc666cc1 100644 --- a/compliance_checker/cf/cf_1_8.py +++ b/compliance_checker/cf/cf_1_8.py @@ -1,16 +1,3 @@ -from compliance_checker.base import BaseCheck, TestCtx -from compliance_checker import MemoizedDataset -from compliance_checker.cf.cf_1_7 import CF1_7Check -from netCDF4 import Dataset -import requests -from lxml import etree -from shapely.geometry import Polygon -import numpy as np -import re -from compliance_checker.cf.util import reference_attr_variables, string_from_var_type -import itertools -import warnings - """ What's new in CF-1.8 -------------------- @@ -23,6 +10,21 @@ 7.5. Geometries """ +import itertools +import re +import warnings + +import numpy as np +import requests +from lxml import etree +from netCDF4 import Dataset +from shapely.geometry import Polygon + +from compliance_checker import MemoizedDataset +from compliance_checker.base import BaseCheck, TestCtx +from compliance_checker.cf.cf_1_7 import CF1_7Check +from compliance_checker.cf.util import reference_attr_variables, string_from_var_type + class CF1_8Check(CF1_7Check): """Implementation for CF v1.8. Inherits from CF1_7Check.""" @@ -249,9 +251,7 @@ def match_taxa_standard_names(standard_name_string): return ( standard_name_string is not None and "taxon" in standard_name_string - and - # exclude the identifiers we just looked at - standard_name_string + and standard_name_string # exclude the identifiers we just looked at not in {"biological_taxon_lsid", "biological_taxon_name"} and standard_name_string in self._std_names ) @@ -400,7 +400,7 @@ def handle_lsid(self, taxon_lsid_variable, taxon_name_variable): timeout=15, ) response.raise_for_status() - except requests.exceptions.RequestException as e: + except requests.exceptions.RequestException as e: # noqa: F841 messages.append( "Aphia ID {taxon_match['object_id'] returned " "other error: {str(e)}" @@ -471,267 +471,6 @@ def handle_lsid(self, taxon_lsid_variable, taxon_name_variable): return messages -class GeometryStorage(object): - """Abstract base class for geometries""" - - def __init__(self, coord_vars, node_count): - self.coord_vars = coord_vars - self.node_count = node_count - self.errors = [] - # geometry is later parsed after sanity checks are run - self.geometry = None - - def _split_mulitpart_geometry(self): - arr_extents_filt = self.part_node_count[self.part_node_count > 0] - splits = np.split(np.vstack(self.coord_vars).T, arr_extents_filt.cumsum()[:-1]) - return splits - - -class PointGeometry(GeometryStorage): - """Class for validating Point/MultiPoint geometries""" - - def check_geometry(self): - super().check_geometry() - # non-multipoint should have exactly one feature - if self.node_count is None: - pass - else: - self.node_count - - if all(len(cv.dimensions) != 0 for cv in self.coord_vars): - same_dim_group = itertools.groupby(self.coord_vars, lambda x: x.dimensions) - same_dim = next(same_dim_group, True) and not next(same_dim_group, False) - if not same_dim: - self.errors.append( - "For a point geometry, coordinate " - "variables must be the same length as " - "node_count defined, or must be " - "length 1 if node_count is not set" - ) - return self.errors - - -class LineGeometry(GeometryStorage): - """Class for validating Line/MultiLine geometries""" - - def __init__(self, coord_vars, node_count, part_node_count): - super().__init__(coord_vars, node_count) - self.part_node_count = part_node_count - if not np.issubdtype(self.node_count.dtype, np.integer): - raise TypeError("For line geometries, node_count must be an integer") - - def check_geometry(self): - geom_errors = [] - same_dim_group = itertools.groupby(self.coord_vars, lambda x: x.dimensions) - same_dim = next(same_dim_group, True) and not next(same_dim_group, False) - if not same_dim: - raise IndexError( - "Coordinate variables must be the same length. " - "If node_count is specified, this value must " - "also sum to the length of the coordinate " - "variables." - ) - # if a multipart - if self.node_count is not None: - same_length = len(self.coord_vars[0]) == self.node_count[:].sum() - if not same_length: - geom_errors.append( - "Coordinate variables must be the same " - "length. If node_count is specified, this " - "value must also sum to the length of the " - "coordinate variables." - ) - if self.part_node_count is not None: - if not np.issubdtype(self.part_node_count.dtype, np.integer): - geom_errors.append( - "when part_node_count is specified, it must " - "be an array of integers" - ) - same_node_count = len(self.coord_vars[0]) == self.node_count[:].sum() - if not same_node_count: - geom_errors.append( - "The sum of part_node_count must be equal " - "to the value of node_count" - ) - return geom_errors - - -class PolygonGeometry(LineGeometry): - """Class for validating Line/MultiLine geometries""" - - # TODO/clarify: Should polygons be simple, i.e. non-self intersecting? - # Presumably - def __init__(self, coord_vars, node_count, part_node_count, interior_ring): - super().__init__(coord_vars, node_count, part_node_count) - self.part_node_count = part_node_count - self.interior_ring = interior_ring - - def check_polygon_orientation(self, transposed_coords, interior=False): - """ - Checks that the polygon orientation is counter-clockwise if an - exterior ring, otherwise clockwise if an interior ring. Orientation - is indicated by the `interior` boolean variable with False for an - exterior ring and True for an interior ring (hole), defaulting to False. - This function operates piecewise on individual interior/exterior - polygons as well as multipart polygons - :param np.array transposed_coords: A 2-by-n array of x and y coordinates - :param bool interior: A boolean defaulting to False which has False - indicating a counter-clockwise or exterior polygon, and True - indicating a clockwise or interior polygon. - :rtype bool: - :returns: True if the polygon follows the proper orientation, - False if it fails the orientation test. - """ - - try: - polygon = Polygon(transposed_coords.tolist()) - except ValueError: - raise ValueError( - "Polygon contains too few points to perform orientation test" - ) - - ccw = polygon.exterior.is_ccw - return not ccw if interior else ccw - - def check_geometry(self): - messages = super().check_geometry() - # If any errors occurred within the preliminary checks, they preclude - # running checks against the geometry here. - if messages: - return messages - if self.part_node_count is not None: - extents = np.concatenate([np.array([0]), self.part_node_count[:].cumsum()]) - if self.interior_ring is not None: - ring_orientation = self.interior_ring[:].astype(bool) - else: - ring_orientation = np.zeros(len(self.part_count), dtype=bool) - node_indexer_len = len(self.part_node_count) - else: - extents = np.concatenate([np.array([0]), self.node_count[:].cumsum()]) - node_indexer_len = len(self.node_count) - ring_orientation = np.zeros(node_indexer_len, dtype=bool) - # TODO: is it necessary to check whether part_node_count "consumes" - # node_count in the polygon, i.e. first (3, 3, 3) will consume - # a node part of 9, follow by next 3 will consume a node part of - # 3 after consuming - for i in range(node_indexer_len): - extent_slice = slice(extents[i], extents[i + 1]) - poly_sliced = np.vstack([cv[extent_slice] for cv in self.coord_vars]).T - pass_orientation = self.check_polygon_orientation( - poly_sliced, ring_orientation[i] - ) - if not pass_orientation: - orient_fix = ( - ("exterior", "counterclockwise") - if not ring_orientation[i] - else ("interior", "clockwise") - ) - message = ( - f"An {orient_fix[0]} polygon referred to by " - f"coordinates ({poly_sliced}) must have coordinates " - f"in {orient_fix[1]} order" - ) - messages.append(message) - return messages - - def check_geometry(self, ds: Dataset): - """Runs any necessary checks for geometry well-formedness - :param netCDF4.Dataset ds: An open netCDF dataset - :returns list: List of error messages - - """ - vars_with_geometry = ds.get_variables_by_attributes( - geometry=lambda g: g is not None - ) - results = [] - unique_geometry_var_names = {var.geometry for var in vars_with_geometry} - if unique_geometry_var_names: - geom_valid = TestCtx(BaseCheck.MEDIUM, self.section_titles["7.5"]) - geom_valid.out_of += 1 - for geometry_var_name in unique_geometry_var_names: - if geometry_var_name not in ds.variables: - geom_valid.messages.append( - "Cannot find geometry variable " f"named {geometry_var_name}" - ) - results.append(geom_valid.to_result()) - continue - else: - geometry_var = ds.variables[geometry_var_name] - - geometry_type = getattr(geometry_var, "geometry_type") - try: - node_coord_var_names = geometry_var.node_coordinates - except AttributeError: - geom_valid.messages.append( - "Could not find required attribute " - '"node_coordinates" in geometry ' - f'variable "{geometry_var_name}"' - ) - results.append(geom_valid.to_result()) - continue - if not isinstance(node_coord_var_names, str): - geom_valid.messages.append( - 'Attribute "node_coordinates" in geometry ' - f'variable "{geometry_var_name}" must be ' - "a string" - ) - results.append(geom_valid.to_result()) - continue - split_coord_names = node_coord_var_names.strip().split(" ") - node_coord_vars, not_found_node_vars = [], [] - for coord_var_name in split_coord_names: - try: - node_coord_vars.append(ds.variables[coord_var_name]) - except KeyError: - not_found_node_vars.append(coord_var_name) - # If any variables weren't found, we can't continue - if not_found_node_vars: - geom_valid.messages.append( - "The following referenced node coordinate" - "variables for geometry variable" - f'"{geometry_var_name}" were not found: ' - f"{not_found_node_vars}" - ) - results.append(geom_valid.to_result()) - continue - - node_count = reference_attr_variables( - ds, getattr(geometry_var, "node_count", None) - ) - # multipart lines and polygons only - part_node_count = reference_attr_variables( - ds, getattr(geometry_var, "part_node_count", None) - ) - # polygons with interior geometry only - interior_ring = reference_attr_variables( - ds, getattr(geometry_var, "interior_ring", None) - ) - - if geometry_type == "point": - geometry = PointGeometry(node_coord_vars, node_count) - elif geometry_type == "line": - geometry = LineGeometry(node_coord_vars, node_count, part_node_count) - elif geometry_type == "polygon": - geometry = PolygonGeometry( - node_coord_vars, node_count, part_node_count, interior_ring - ) - else: - geom_valid.messages.append( - f'For geometry variable "{geometry_var_name}' - 'the attribute "geometry_type" must exist' - "and have one of the following values:" - '"point", "line", "polygon"' - ) - results.append(geom_valid.to_result()) - continue - # check geometry - geometry.check_geometry() - if not geometry.errors: # geom_valid.messages: - geom_valid.score += 1 - results.append(geom_valid.to_result()) - return results - - class GeometryStorage(object): """Abstract base class for geometries""" diff --git a/compliance_checker/cf/cf_base.py b/compliance_checker/cf/cf_base.py index 895d2089..07f0ca70 100644 --- a/compliance_checker/cf/cf_base.py +++ b/compliance_checker/cf/cf_base.py @@ -3,20 +3,16 @@ import logging import os import sys - from collections import OrderedDict, defaultdict from warnings import warn import numpy as np import regex - from compliance_checker import cfutil from compliance_checker.base import BaseCheck, BaseNCCheck, Result, TestCtx from compliance_checker.cf import util -from compliance_checker.cf.appendix_d import ( - no_missing_terms, -) +from compliance_checker.cf.appendix_d import no_missing_terms logger = logging.getLogger(__name__) diff --git a/compliance_checker/cf/util.py b/compliance_checker/cf/util.py index e36966fe..e600fa99 100644 --- a/compliance_checker/cf/util.py +++ b/compliance_checker/cf/util.py @@ -2,7 +2,6 @@ import itertools import os import sys - from collections import defaultdict from copy import deepcopy from pkgutil import get_data @@ -10,13 +9,11 @@ import lxml.html import requests - from cf_units import Unit from lxml import etree from netCDF4 import Dataset, Dimension, Variable from pkg_resources import resource_filename - # copied from paegan # paegan may depend on these later _possiblet = { diff --git a/compliance_checker/cfutil.py b/compliance_checker/cfutil.py index da5f8390..a5c322bb 100644 --- a/compliance_checker/cfutil.py +++ b/compliance_checker/cfutil.py @@ -6,14 +6,12 @@ import csv import re import warnings - from collections import defaultdict from functools import lru_cache, partial from cf_units import Unit from pkg_resources import resource_filename - _UNITLESS_DB = None _SEA_NAMES = None diff --git a/compliance_checker/ioos.py b/compliance_checker/ioos.py index 6c9a1654..73c2fbee 100644 --- a/compliance_checker/ioos.py +++ b/compliance_checker/ioos.py @@ -2,11 +2,9 @@ Check for IOOS-approved attributes """ import re - from numbers import Number import validators - from cf_units import Unit from lxml.etree import XPath from owslib.namespaces import Namespaces @@ -426,10 +424,9 @@ class NamingAuthorityValidator(base.UrlValidator): def validator_func(self, input_value): return ( + # also check for reverse DNS strings super().validator_func(input_value) - or - # check for reverse DNS strings - validators.domain(".".join(input_value.split(".")[::-1])) + or validators.domain(".".join(input_value.split(".")[::-1])) ) diff --git a/compliance_checker/protocols/netcdf.py b/compliance_checker/protocols/netcdf.py index 65c58748..519f132d 100644 --- a/compliance_checker/protocols/netcdf.py +++ b/compliance_checker/protocols/netcdf.py @@ -5,6 +5,8 @@ Functions to assist in determining if the URL points to a netCDF file """ +import warnings + import requests @@ -79,7 +81,10 @@ def is_remote_netcdf(ds_str): try: head_req = requests.head(ds_str, allow_redirects=True, timeout=10) head_req.raise_for_status() - except: + except requests.exceptions.RequestException as e: + warnings.warn( + "Received exception when making HEAD request to {}: {}".format(ds_str, e) + ) content_type = None else: content_type = head_req.headers.get("content-type") diff --git a/compliance_checker/runner.py b/compliance_checker/runner.py index 9755318d..928aee3a 100644 --- a/compliance_checker/runner.py +++ b/compliance_checker/runner.py @@ -3,7 +3,6 @@ import os import sys import traceback - from collections import OrderedDict from contextlib import contextmanager diff --git a/compliance_checker/suite.py b/compliance_checker/suite.py index 35cf82b2..ca3d0782 100644 --- a/compliance_checker/suite.py +++ b/compliance_checker/suite.py @@ -11,16 +11,14 @@ import sys import textwrap import warnings - from collections import defaultdict from datetime import datetime, timezone -from distutils.version import StrictVersion from operator import itemgetter from pathlib import Path from urllib.parse import urlparse import requests - +from distutils.version import StrictVersion from lxml import etree as ET from netCDF4 import Dataset from owslib.sos import SensorObservationService @@ -31,7 +29,6 @@ from compliance_checker.base import BaseCheck, GenericFile, Result, fix_return_value from compliance_checker.protocols import cdl, netcdf, opendap - # Ensure output is encoded as Unicode when checker output is redirected or piped if sys.stdout.encoding is None: sys.stdout = codecs.getwriter("utf8")(sys.stdout) diff --git a/compliance_checker/tests/conftest.py b/compliance_checker/tests/conftest.py index 94e8e5c0..c83e5f44 100644 --- a/compliance_checker/tests/conftest.py +++ b/compliance_checker/tests/conftest.py @@ -1,11 +1,9 @@ import os import subprocess - from itertools import chain from pathlib import Path import pytest - from netCDF4 import Dataset from pkg_resources import resource_filename diff --git a/compliance_checker/tests/test_acdd.py b/compliance_checker/tests/test_acdd.py index 4e9ea6e8..453ce58c 100644 --- a/compliance_checker/tests/test_acdd.py +++ b/compliance_checker/tests/test_acdd.py @@ -1,7 +1,6 @@ import os import numpy as np - from netCDF4 import Dataset from compliance_checker.acdd import ACDD1_1Check, ACDD1_3Check @@ -15,7 +14,9 @@ def to_singleton_var(item): Get the first value of a list if this implements iterator protocol and is not a string """ - return [x[0] if hasattr(x, "__iter__") and not isinstance(x, str) else x for x in item] + return [ + x[0] if hasattr(x, "__iter__") and not isinstance(x, str) else x for x in item + ] def check_varset_nonintersect(group0, group1): diff --git a/compliance_checker/tests/test_base.py b/compliance_checker/tests/test_base.py index e0faf557..87b30735 100644 --- a/compliance_checker/tests/test_base.py +++ b/compliance_checker/tests/test_base.py @@ -3,7 +3,6 @@ """Tests for base compliance checker class""" import os - from unittest import TestCase from netCDF4 import Dataset diff --git a/compliance_checker/tests/test_cf.py b/compliance_checker/tests/test_cf.py index 9fd18185..d4563794 100644 --- a/compliance_checker/tests/test_cf.py +++ b/compliance_checker/tests/test_cf.py @@ -2,18 +2,20 @@ # -*- coding: utf-8 -*- import copy +import json import os +import re import sqlite3 - from itertools import chain from tempfile import gettempdir import numpy as np import pytest - +import requests_mock from netCDF4 import Dataset, stringtoarr from compliance_checker import cfutil +from compliance_checker.cf.appendix_d import no_missing_terms from compliance_checker.cf.cf import ( CF1_6Check, CF1_7Check, @@ -21,7 +23,6 @@ dimless_vertical_coordinates_1_6, dimless_vertical_coordinates_1_7, ) -from compliance_checker.cf.appendix_d import no_missing_terms from compliance_checker.cf.util import ( StandardNameTable, create_cached_data_dir, @@ -38,9 +39,6 @@ MockTimeSeries, MockVariable, ) -import requests_mock -import json -import re from compliance_checker.tests.resources import STATIC_FILES diff --git a/compliance_checker/tests/test_cf_integration.py b/compliance_checker/tests/test_cf_integration.py index b6a01b9c..ab1e849d 100644 --- a/compliance_checker/tests/test_cf_integration.py +++ b/compliance_checker/tests/test_cf_integration.py @@ -3,10 +3,8 @@ import pytest - from compliance_checker.cf import util - # get current std names table version (it changes) std_names = util.StandardNameTable() diff --git a/compliance_checker/tests/test_cli.py b/compliance_checker/tests/test_cli.py index c12c3c1c..42682d9b 100644 --- a/compliance_checker/tests/test_cli.py +++ b/compliance_checker/tests/test_cli.py @@ -8,7 +8,6 @@ import json import os import sys - from argparse import Namespace import pytest diff --git a/compliance_checker/tests/test_ioos_profile.py b/compliance_checker/tests/test_ioos_profile.py index 2ecad34e..9f8d2db9 100644 --- a/compliance_checker/tests/test_ioos_profile.py +++ b/compliance_checker/tests/test_ioos_profile.py @@ -1,7 +1,6 @@ import os import numpy as np - from netCDF4 import Dataset from compliance_checker.ioos import ( diff --git a/compliance_checker/tests/test_ioos_sos.py b/compliance_checker/tests/test_ioos_sos.py index 9f360f36..8b82f1f5 100644 --- a/compliance_checker/tests/test_ioos_sos.py +++ b/compliance_checker/tests/test_ioos_sos.py @@ -27,9 +27,9 @@ def test_retrieve_getcaps(self): httpretty.register_uri( httpretty.GET, url, content_type="text/xml", body=self.resp ) - # need to mock out the HEAD response so that compliance checker - # recognizes this as some sort of XML doc instead of an OPeNDAP - # source + httpretty.register_uri( + httpretty.HEAD, url, content_type="text/xml", body="HTTP/1.1 200" + ) ComplianceChecker.run_checker(url, ["ioos_sos"], 1, "normal") @@ -59,6 +59,9 @@ def test_retrieve_describesensor(self): httpretty.register_uri( httpretty.GET, url, content_type="text/xml", body=self.resp ) + httpretty.register_uri( + httpretty.HEAD, url, content_type="text/xml", body="HTTP/1.1 200" + ) # need to mock out the HEAD response so that compliance checker # recognizes this as some sort of XML doc instead of an OPeNDAP # source diff --git a/compliance_checker/tests/test_suite.py b/compliance_checker/tests/test_suite.py index 325c1f86..bc79b0a9 100644 --- a/compliance_checker/tests/test_suite.py +++ b/compliance_checker/tests/test_suite.py @@ -1,17 +1,14 @@ # coding=utf-8 import os import unittest - from pathlib import Path import numpy as np - from pkg_resources import resource_filename from compliance_checker.base import BaseCheck, GenericFile, Result from compliance_checker.suite import CheckSuite - static_files = { "2dim": resource_filename("compliance_checker", "tests/data/2dim-grid.nc"), "bad_region": resource_filename("compliance_checker", "tests/data/bad_region.nc"), @@ -224,7 +221,7 @@ def test_cdl_file(self): def test_load_local_dataset_GenericFile(self): resp = self.cs.load_local_dataset(static_files["empty"]) - assert isinstance(resp, GenericFile) == True + assert isinstance(resp, GenericFile) def test_standard_output_score_header(self): """ diff --git a/compliance_checker/util.py b/compliance_checker/util.py index a9bb16ee..8117ad3e 100644 --- a/compliance_checker/util.py +++ b/compliance_checker/util.py @@ -15,7 +15,8 @@ def datetime_is_iso(date_str): else: isodate.parse_date(date_str) return True, [] - except: # Any error qualifies as not ISO format + # The following errors qualify as non ISO8601 format + except (TypeError, AttributeError, ValueError, isodate.ISO8601Error): return False, ["Datetime provided is not in a valid ISO 8601 format"] @@ -43,6 +44,6 @@ def kvp_convert(input_coll): return input_coll else: return OrderedDict( - (thing, None) if not isinstance(thing, tuple) else - (thing[0], thing[1]) for thing in input_coll + (thing, None) if not isinstance(thing, tuple) else (thing[0], thing[1]) + for thing in input_coll ) diff --git a/pyproject.toml b/pyproject.toml index 7825e3d5..3856b400 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,3 +7,6 @@ markers = [ "integration: marks integration tests (deselect with '-m \"not integration\"')", "slowtest: marks slow tests (deselect with '-m \"not slowtest\"')" ] + +[tool.isort] +profile = "black" diff --git a/requirements.txt b/requirements.txt index 5a591098..144e32d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,5 +11,5 @@ pyproj>=2.2.1 regex>=2017.07.28 requests>=2.2.1 setuptools>=15.0 -validators>=0.14.2 shapely>=1.7.1 +validators>=0.14.2 diff --git a/test_requirements.txt b/test_requirements.txt index 656a55f1..4875429d 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,9 +1,9 @@ +codecov codespell flake8 httpretty mypy pre-commit pytest>=2.9.0 +pytest-cov>=3.0.0 requests-mock>=1.7.0 -codecov -pytest-cov