diff --git a/CHANGELOG.md b/CHANGELOG.md index a87083885..1f9a9c864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Splunk SDK for Python Changelog +## Version 1.6.11 + +### Bug Fix + +* Fix custom search command V2 failures on Windows for Python3 + ## Version 1.6.10 ### Bug Fix diff --git a/README.md b/README.md index 5ca083dd4..d4b819cde 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # The Splunk Software Development Kit for Python -#### Version 1.6.10 +#### Version 1.6.11 The Splunk Software Development Kit (SDK) for Python contains library code and examples designed to enable developers to build applications using Splunk. diff --git a/examples/searchcommands_app/setup.py b/examples/searchcommands_app/setup.py index 170949978..b84d5bbb4 100755 --- a/examples/searchcommands_app/setup.py +++ b/examples/searchcommands_app/setup.py @@ -439,7 +439,7 @@ def run(self): setup( description='Custom Search Command examples', name=os.path.basename(project_dir), - version='1.6.10', + version='1.6.11', author='Splunk, Inc.', author_email='devinfo@splunk.com', url='http://github.com/splunk/splunk-sdk-python', diff --git a/splunklib/__init__.py b/splunklib/__init__.py index 9b7cc5f26..7b3767120 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -16,5 +16,5 @@ from __future__ import absolute_import from splunklib.six.moves import map -__version_info__ = (1, 6, 10) +__version_info__ = (1, 6, 11) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index 1cd2017e8..bfccf6f3b 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -1369,7 +1369,7 @@ def request(url, message, **kwargs): head = { "Content-Length": str(len(body)), "Host": host, - "User-Agent": "splunk-sdk-python/1.6.10", + "User-Agent": "splunk-sdk-python/1.6.11", "Accept": "*/*", "Connection": "Close", } # defaults diff --git a/splunklib/searchcommands/internals.py b/splunklib/searchcommands/internals.py index 2f22cfe40..9a2a4e9e2 100644 --- a/splunklib/searchcommands/internals.py +++ b/splunklib/searchcommands/internals.py @@ -16,6 +16,7 @@ from __future__ import absolute_import, division, print_function +from io import TextIOWrapper from collections import deque, namedtuple from splunklib import six try: @@ -39,19 +40,37 @@ csv.field_size_limit(10485760) # The default value is 128KB; upping to 10MB. See SPL-12117 for background on this issue -# SPL-175233 -- python3 stdout is already binary -if sys.platform == 'win32' and sys.version_info <= (3, 0): - # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in - # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that - # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption. - from platform import python_implementation - implementation = python_implementation() - fileno = sys.stdout.fileno() - if implementation == 'PyPy': - sys.stdout = os.fdopen(fileno, 'wb', 0) - else: - from msvcrt import setmode - setmode(fileno, os.O_BINARY) + +def set_binary_mode(fh): + """ Helper method to set up binary mode for file handles. + Emphasis being sys.stdin, sys.stdout, sys.stderr. + For python3, we want to return .buffer + For python2+windows we want to set os.O_BINARY + """ + typefile = TextIOWrapper if sys.version_info >= (3, 0) else file + # check for file handle + if not isinstance(fh, typefile): + return fh + + # check for python3 and buffer + if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'): + return fh.buffer + # check for python3 + elif sys.version_info >= (3, 0): + pass + # check for windows python2. SPL-175233 -- python3 stdout is already binary + elif sys.platform == 'win32': + # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in + # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that + # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption. + from platform import python_implementation + implementation = python_implementation() + if implementation == 'PyPy': + return os.fdopen(fh.fileno(), 'wb', 0) + else: + import msvcrt + msvcrt.setmode(fh.fileno(), os.O_BINARY) + return fh class CommandLineParser(object): @@ -349,6 +368,7 @@ class InputHeader(dict): """ Represents a Splunk input header as a collection of name/value pairs. """ + def __str__(self): return '\n'.join([name + ':' + value for name, value in six.iteritems(self)]) @@ -376,7 +396,8 @@ def read(self, ifile): # continuation of the current item value += urllib.parse.unquote(line) - if name is not None: self[name] = value[:-1] if value[-1] == '\n' else value + if name is not None: + self[name] = value[:-1] if value[-1] == '\n' else value Message = namedtuple('Message', ('type', 'text')) @@ -473,7 +494,7 @@ class RecordWriter(object): def __init__(self, ofile, maxresultrows=None): self._maxresultrows = 50000 if maxresultrows is None else maxresultrows - self._ofile = ofile + self._ofile = set_binary_mode(ofile) self._fieldnames = None self._buffer = StringIO() @@ -501,7 +522,13 @@ def ofile(self): @ofile.setter def ofile(self, value): - self._ofile = value + self._ofile = set_binary_mode(value) + + def write(self, data): + bytes_type = bytes if sys.version_info >= (3, 0) else str + if not isinstance(data, bytes_type): + data = data.encode('utf-8') + self.ofile.write(data) def flush(self, finished=None, partial=None): assert finished is None or isinstance(finished, bool) @@ -661,19 +688,11 @@ class RecordWriterV1(RecordWriter): def flush(self, finished=None, partial=None): - # SPL-175233 - def writeEOL(): - if sys.version_info >= (3, 0) and sys.platform == 'win32': - write('\n') - else: - write('\r\n') - RecordWriter.flush(self, finished, partial) # validates arguments and the state of this instance if self._record_count > 0 or (self._chunk_count == 0 and 'messages' in self._inspector): messages = self._inspector.get('messages') - write = self._ofile.write if self._chunk_count == 0: @@ -685,12 +704,12 @@ def writeEOL(): message_level = RecordWriterV1._message_level.get for level, text in messages: - write(message_level(level, level)) - write('=') - write(text) - writeEOL() + self.write(message_level(level, level)) + self.write('=') + self.write(text) + self.write('\r\n') - writeEOL() + self.write('\r\n') elif messages is not None: @@ -708,7 +727,7 @@ def writeEOL(): for level, text in messages: print(level, text, file=stderr) - write(self._buffer.getvalue()) + self.write(self._buffer.getvalue()) self._clear() self._chunk_count += 1 self._total_record_count += self._record_count @@ -766,7 +785,7 @@ def write_metadata(self, configuration): metadata = chain(six.iteritems(configuration), (('inspector', self._inspector if self._inspector else None),)) self._write_chunk(metadata, '') - self._ofile.write('\n') + self.write('\n') self._clear() def write_metric(self, name, value): @@ -781,19 +800,22 @@ def _write_chunk(self, metadata, body): if metadata: metadata = str(''.join(self._iterencode_json(dict([(n, v) for n, v in metadata if v is not None]), 0))) + if sys.version_info >= (3, 0): + metadata = metadata.encode('utf-8') metadata_length = len(metadata) else: metadata_length = 0 + if sys.version_info >= (3, 0): + body = body.encode('utf-8') body_length = len(body) if not (metadata_length > 0 or body_length > 0): return start_line = 'chunked 1.0,%s,%s\n' % (metadata_length, body_length) - write = self._ofile.write - write(start_line) - write(metadata) - write(body) + self.write(start_line) + self.write(metadata) + self.write(body) self._ofile.flush() self._flushed = False