From 3ce5f5633b7155d9623fa84036af0b233a23026a Mon Sep 17 00:00:00 2001 From: Junyu Wang Date: Fri, 4 Oct 2019 14:24:32 -0700 Subject: [PATCH 1/3] ITSI-4561: fix sdk py3 --- splunklib/searchcommands/internals.py | 35 +++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/splunklib/searchcommands/internals.py b/splunklib/searchcommands/internals.py index 2f22cfe40..d4787407a 100644 --- a/splunklib/searchcommands/internals.py +++ b/splunklib/searchcommands/internals.py @@ -16,6 +16,7 @@ from __future__ import absolute_import, division, print_function +from io import TextIOWrapper from collections import deque, namedtuple from splunklib import six try: @@ -781,19 +782,49 @@ def _write_chunk(self, metadata, body): if metadata: metadata = str(''.join(self._iterencode_json(dict([(n, v) for n, v in metadata if v is not None]), 0))) + if sys.version_info >= (3, 0): + metadata = metadata.encode('utf-8') metadata_length = len(metadata) else: metadata_length = 0 + if sys.version_info >= (3, 0): + body = body.encode('utf-8') body_length = len(body) if not (metadata_length > 0 or body_length > 0): return start_line = 'chunked 1.0,%s,%s\n' % (metadata_length, body_length) - write = self._ofile.write - write(start_line) + ofile_handle = set_binary_mode(self._ofile) + write = ofile_handle.write + write(start_line.encode('utf-8')) write(metadata) write(body) self._ofile.flush() self._flushed = False + +def set_binary_mode(fh): + """ Helper method to set up binary mode for file handles. + Emphasis being sys.stdin, sys.stdout, sys.stderr. + For python3, we want to return .buffer + For python2+windows we want to set os.O_BINARY + """ + typefile = TextIOWrapper if sys.version_info >= (3, 0) else file + # check for file handle + if not isinstance(fh, typefile): + return fh + + # check for python3 and buffer + if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'): + return fh.buffer + # check for python3 + elif sys.version_info >= (3, 0): + pass + # check for windows + elif sys.platform == 'win32': + import msvcrt + msvcrt.setmode(fh.fileno(), os.O_BINARY) + + return fh + From 189ccfe9d671014a630eeb5b9362d19691a77dd1 Mon Sep 17 00:00:00 2001 From: Junyu Wang Date: Sat, 5 Oct 2019 16:13:42 -0700 Subject: [PATCH 2/3] DVPL-7646: fix generic ofile binary write issue --- splunklib/searchcommands/internals.py | 113 ++++++++++++-------------- 1 file changed, 52 insertions(+), 61 deletions(-) diff --git a/splunklib/searchcommands/internals.py b/splunklib/searchcommands/internals.py index d4787407a..9a2a4e9e2 100644 --- a/splunklib/searchcommands/internals.py +++ b/splunklib/searchcommands/internals.py @@ -40,19 +40,37 @@ csv.field_size_limit(10485760) # The default value is 128KB; upping to 10MB. See SPL-12117 for background on this issue -# SPL-175233 -- python3 stdout is already binary -if sys.platform == 'win32' and sys.version_info <= (3, 0): - # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in - # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that - # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption. - from platform import python_implementation - implementation = python_implementation() - fileno = sys.stdout.fileno() - if implementation == 'PyPy': - sys.stdout = os.fdopen(fileno, 'wb', 0) - else: - from msvcrt import setmode - setmode(fileno, os.O_BINARY) + +def set_binary_mode(fh): + """ Helper method to set up binary mode for file handles. + Emphasis being sys.stdin, sys.stdout, sys.stderr. + For python3, we want to return .buffer + For python2+windows we want to set os.O_BINARY + """ + typefile = TextIOWrapper if sys.version_info >= (3, 0) else file + # check for file handle + if not isinstance(fh, typefile): + return fh + + # check for python3 and buffer + if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'): + return fh.buffer + # check for python3 + elif sys.version_info >= (3, 0): + pass + # check for windows python2. SPL-175233 -- python3 stdout is already binary + elif sys.platform == 'win32': + # Work around the fact that on Windows '\n' is mapped to '\r\n'. The typical solution is to simply open files in + # binary mode, but stdout is already open, thus this hack. 'CPython' and 'PyPy' work differently. We assume that + # all other Python implementations are compatible with 'CPython'. This might or might not be a valid assumption. + from platform import python_implementation + implementation = python_implementation() + if implementation == 'PyPy': + return os.fdopen(fh.fileno(), 'wb', 0) + else: + import msvcrt + msvcrt.setmode(fh.fileno(), os.O_BINARY) + return fh class CommandLineParser(object): @@ -350,6 +368,7 @@ class InputHeader(dict): """ Represents a Splunk input header as a collection of name/value pairs. """ + def __str__(self): return '\n'.join([name + ':' + value for name, value in six.iteritems(self)]) @@ -377,7 +396,8 @@ def read(self, ifile): # continuation of the current item value += urllib.parse.unquote(line) - if name is not None: self[name] = value[:-1] if value[-1] == '\n' else value + if name is not None: + self[name] = value[:-1] if value[-1] == '\n' else value Message = namedtuple('Message', ('type', 'text')) @@ -474,7 +494,7 @@ class RecordWriter(object): def __init__(self, ofile, maxresultrows=None): self._maxresultrows = 50000 if maxresultrows is None else maxresultrows - self._ofile = ofile + self._ofile = set_binary_mode(ofile) self._fieldnames = None self._buffer = StringIO() @@ -502,7 +522,13 @@ def ofile(self): @ofile.setter def ofile(self, value): - self._ofile = value + self._ofile = set_binary_mode(value) + + def write(self, data): + bytes_type = bytes if sys.version_info >= (3, 0) else str + if not isinstance(data, bytes_type): + data = data.encode('utf-8') + self.ofile.write(data) def flush(self, finished=None, partial=None): assert finished is None or isinstance(finished, bool) @@ -662,19 +688,11 @@ class RecordWriterV1(RecordWriter): def flush(self, finished=None, partial=None): - # SPL-175233 - def writeEOL(): - if sys.version_info >= (3, 0) and sys.platform == 'win32': - write('\n') - else: - write('\r\n') - RecordWriter.flush(self, finished, partial) # validates arguments and the state of this instance if self._record_count > 0 or (self._chunk_count == 0 and 'messages' in self._inspector): messages = self._inspector.get('messages') - write = self._ofile.write if self._chunk_count == 0: @@ -686,12 +704,12 @@ def writeEOL(): message_level = RecordWriterV1._message_level.get for level, text in messages: - write(message_level(level, level)) - write('=') - write(text) - writeEOL() + self.write(message_level(level, level)) + self.write('=') + self.write(text) + self.write('\r\n') - writeEOL() + self.write('\r\n') elif messages is not None: @@ -709,7 +727,7 @@ def writeEOL(): for level, text in messages: print(level, text, file=stderr) - write(self._buffer.getvalue()) + self.write(self._buffer.getvalue()) self._clear() self._chunk_count += 1 self._total_record_count += self._record_count @@ -767,7 +785,7 @@ def write_metadata(self, configuration): metadata = chain(six.iteritems(configuration), (('inspector', self._inspector if self._inspector else None),)) self._write_chunk(metadata, '') - self._ofile.write('\n') + self.write('\n') self._clear() def write_metric(self, name, value): @@ -796,35 +814,8 @@ def _write_chunk(self, metadata, body): return start_line = 'chunked 1.0,%s,%s\n' % (metadata_length, body_length) - ofile_handle = set_binary_mode(self._ofile) - write = ofile_handle.write - write(start_line.encode('utf-8')) - write(metadata) - write(body) + self.write(start_line) + self.write(metadata) + self.write(body) self._ofile.flush() self._flushed = False - -def set_binary_mode(fh): - """ Helper method to set up binary mode for file handles. - Emphasis being sys.stdin, sys.stdout, sys.stderr. - For python3, we want to return .buffer - For python2+windows we want to set os.O_BINARY - """ - typefile = TextIOWrapper if sys.version_info >= (3, 0) else file - # check for file handle - if not isinstance(fh, typefile): - return fh - - # check for python3 and buffer - if sys.version_info >= (3, 0) and hasattr(fh, 'buffer'): - return fh.buffer - # check for python3 - elif sys.version_info >= (3, 0): - pass - # check for windows - elif sys.platform == 'win32': - import msvcrt - msvcrt.setmode(fh.fileno(), os.O_BINARY) - - return fh - From bb2bec332b8dd4d751f0e95a2c046ff1c75dc510 Mon Sep 17 00:00:00 2001 From: Liying Jiang Date: Tue, 8 Oct 2019 12:54:03 -0700 Subject: [PATCH 3/3] update version to 1.6.11 --- CHANGELOG.md | 6 ++++++ README.md | 2 +- examples/searchcommands_app/setup.py | 2 +- splunklib/__init__.py | 2 +- splunklib/binding.py | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a87083885..1f9a9c864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Splunk SDK for Python Changelog +## Version 1.6.11 + +### Bug Fix + +* Fix custom search command V2 failures on Windows for Python3 + ## Version 1.6.10 ### Bug Fix diff --git a/README.md b/README.md index 5ca083dd4..d4b819cde 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # The Splunk Software Development Kit for Python -#### Version 1.6.10 +#### Version 1.6.11 The Splunk Software Development Kit (SDK) for Python contains library code and examples designed to enable developers to build applications using Splunk. diff --git a/examples/searchcommands_app/setup.py b/examples/searchcommands_app/setup.py index 170949978..b84d5bbb4 100755 --- a/examples/searchcommands_app/setup.py +++ b/examples/searchcommands_app/setup.py @@ -439,7 +439,7 @@ def run(self): setup( description='Custom Search Command examples', name=os.path.basename(project_dir), - version='1.6.10', + version='1.6.11', author='Splunk, Inc.', author_email='devinfo@splunk.com', url='http://github.com/splunk/splunk-sdk-python', diff --git a/splunklib/__init__.py b/splunklib/__init__.py index 9b7cc5f26..7b3767120 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -16,5 +16,5 @@ from __future__ import absolute_import from splunklib.six.moves import map -__version_info__ = (1, 6, 10) +__version_info__ = (1, 6, 11) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index 1cd2017e8..bfccf6f3b 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -1369,7 +1369,7 @@ def request(url, message, **kwargs): head = { "Content-Length": str(len(body)), "Host": host, - "User-Agent": "splunk-sdk-python/1.6.10", + "User-Agent": "splunk-sdk-python/1.6.11", "Accept": "*/*", "Connection": "Close", } # defaults