From 681c8d7b770e0f450ecf961d358d02e357fd7423 Mon Sep 17 00:00:00 2001 From: Liam Hopkins Date: Wed, 31 Jul 2019 11:25:06 -0700 Subject: [PATCH 1/5] Correct JSON refcount decrementing (#821) --- packages/google-compute-engine-oslogin/src/utils.cc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/google-compute-engine-oslogin/src/utils.cc b/packages/google-compute-engine-oslogin/src/utils.cc index c97529e2..0f05f6f3 100644 --- a/packages/google-compute-engine-oslogin/src/utils.cc +++ b/packages/google-compute-engine-oslogin/src/utils.cc @@ -800,7 +800,7 @@ bool StartSession(const string& email, string* response) { jobj = json_object_new_object(); json_object_object_add(jobj, "email", json_object_new_string(email.c_str())); - json_object_object_add(jobj, "supportedChallengeTypes", jarr); + json_object_object_add(jobj, "supportedChallengeTypes", jarr); // Ownership transferred to jobj. const char* data; data = json_object_to_json_string_ext(jobj, JSON_C_TO_STRING_PLAIN); @@ -814,7 +814,6 @@ bool StartSession(const string& email, string* response) { ret = false; } - json_object_put(jarr); json_object_put(jobj); return ret; @@ -843,7 +842,7 @@ bool ContinueSession(bool alt, const string& email, const string& user_token, jresp = json_object_new_object(); json_object_object_add(jresp, "credential", json_object_new_string(user_token.c_str())); - json_object_object_add(jobj, "proposalResponse", jresp); + json_object_object_add(jobj, "proposalResponse", jresp); // Ownership transferred to jobj. } const char* data = NULL; @@ -859,10 +858,6 @@ bool ContinueSession(bool alt, const string& email, const string& user_token, } json_object_put(jobj); - // Match condition where we created this to avoid double-free. - if (challenge.type != AUTHZEN && !alt) { - json_object_put(jresp); - } return ret; } From c38dc90bdcc427508a1e69d54575543f7881223a Mon Sep 17 00:00:00 2001 From: Max Illfelder Date: Wed, 31 Jul 2019 16:02:15 -0700 Subject: [PATCH 2/5] Update script retrieval for Python 2 and 3. (#822) Non-breaking spaces cause utf-8 encoding to fail during file writes. Writing as bytes should be 2/3 compatible. --- .../metadata_scripts/script_retriever.py | 4 ++-- .../metadata_scripts/tests/script_retriever_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py index c880f49a..3de425b6 100644 --- a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py +++ b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py @@ -112,12 +112,12 @@ def _DownloadAuthUrl(self, url, dest_dir): request = urlrequest.Request(url) request.add_unredirected_header('Metadata-Flavor', 'Google') request.add_unredirected_header('Authorization', self.token) - content = _UrlOpenWithRetry(request).read().decode('utf-8') + content = _UrlOpenWithRetry(request).read() except Exception as e: self.logger.warning('Could not download %s. %s.', url, str(e)) return None - with open(dest, 'w') as f: + with open(dest, 'wb') as f: f.write(content) return dest diff --git a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py index b9d9d1b2..0df8830f 100644 --- a/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py +++ b/packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py @@ -59,10 +59,10 @@ def testDownloadAuthUrl(self, mock_urlopen, mock_request, mock_tempfile): mocked_request.add_unredirected_header.assert_called_with( 'Authorization', 'bar') mock_urlopen.assert_called_with(mocked_request) - urlopen_read = mock_urlopen().read(return_value=b'foo').decode() + urlopen_read = mock_urlopen().read(return_value=b'foo') self.mock_logger.warning.assert_not_called() - mock_open.assert_called_once_with(self.dest, 'w') + mock_open.assert_called_once_with(self.dest, 'wb') handle = mock_open() handle.write.assert_called_once_with(urlopen_read) From 66b0268f237da0d046a97c6db42071cea1b6efb0 Mon Sep 17 00:00:00 2001 From: Zach Marano Date: Thu, 1 Aug 2019 13:54:13 -0700 Subject: [PATCH 3/5] Re-enable boto config and don't write plugin line. (#823) --- .../google_compute_engine/boto/boto_config.py | 2 -- .../google_compute_engine/boto/tests/boto_config_test.py | 1 - .../google_compute_engine/constants.py | 6 ------ .../google_compute_engine/instance_setup/instance_config.py | 2 +- 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/python-google-compute-engine/google_compute_engine/boto/boto_config.py b/packages/python-google-compute-engine/google_compute_engine/boto/boto_config.py index 6771c5cc..155f3c7e 100644 --- a/packages/python-google-compute-engine/google_compute_engine/boto/boto_config.py +++ b/packages/python-google-compute-engine/google_compute_engine/boto/boto_config.py @@ -80,10 +80,8 @@ def _CreateConfig(self, project_id): config = config_manager.ConfigManager( config_file=self.boto_config_template, config_header=self.boto_config_header) - boto_dir = os.path.dirname(self.boto_config_script) config.SetOption('GSUtil', 'default_project_id', project_id) config.SetOption('GSUtil', 'default_api_version', '2') config.SetOption('GoogleCompute', 'service_account', 'default') - config.SetOption('Plugin', 'plugin_directory', boto_dir) config.WriteConfig(config_file=self.boto_config) diff --git a/packages/python-google-compute-engine/google_compute_engine/boto/tests/boto_config_test.py b/packages/python-google-compute-engine/google_compute_engine/boto/tests/boto_config_test.py index 3ba4168c..bc2c05a1 100644 --- a/packages/python-google-compute-engine/google_compute_engine/boto/tests/boto_config_test.py +++ b/packages/python-google-compute-engine/google_compute_engine/boto/tests/boto_config_test.py @@ -53,7 +53,6 @@ def testCreateConfig(self, mock_config, mock_logger, mock_watcher): mock.call.set('GSUtil', 'default_project_id', self.project_id), mock.call.set('GSUtil', 'default_api_version', '2'), mock.call.set('GoogleCompute', 'service_account', 'default'), - mock.call.set('Plugin', 'plugin_directory', '/tmp'), mock.call.write(config_file='config'), ] self.assertEqual(mocks.mock_calls, expected_calls) diff --git a/packages/python-google-compute-engine/google_compute_engine/constants.py b/packages/python-google-compute-engine/google_compute_engine/constants.py index d929baf8..cfe87c49 100644 --- a/packages/python-google-compute-engine/google_compute_engine/constants.py +++ b/packages/python-google-compute-engine/google_compute_engine/constants.py @@ -16,7 +16,6 @@ """A module for global constants.""" import platform -import sys OSLOGIN_CONTROL_SCRIPT = 'google_oslogin_control' OSLOGIN_NSS_CACHE_SCRIPT = 'google_oslogin_nss_cache' @@ -42,8 +41,3 @@ OSLOGIN_NSS_CACHE = '/etc/oslogin_passwd.cache' SYSCONFDIR = '/etc/default' SYSLOG_SOCKET = '/dev/log' - -if sys.version_info >= (3, 0): - SET_BOTO_CONFIG = 'false' -else: - SET_BOTO_CONFIG = 'true' diff --git a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py index 23261152..1c23171d 100644 --- a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py +++ b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py @@ -78,7 +78,7 @@ class InstanceConfig(config_manager.ConfigManager): 'host_key_types': 'ecdsa,ed25519,rsa', 'optimize_local_ssd': 'true', 'network_enabled': 'true', - 'set_boto_config': constants.SET_BOTO_CONFIG, + 'set_boto_config': 'true', 'set_host_keys': 'true', 'set_multiqueue': 'true', }, From 72bf685fec63cbcd8d40ca8878197419e9649610 Mon Sep 17 00:00:00 2001 From: Zach Marano Date: Thu, 1 Aug 2019 14:08:59 -0700 Subject: [PATCH 4/5] Bump versions. (#825) --- .../packaging/debian/changelog | 6 ++++++ .../google-compute-engine-oslogin/packaging/setup_deb.sh | 2 +- .../google-compute-engine-oslogin/packaging/setup_rpm.sh | 2 +- packages/google-compute-engine-oslogin/src/Makefile | 2 +- packages/google-compute-engine/packaging/debian/changelog | 7 +++++++ packages/google-compute-engine/packaging/setup_deb.sh | 2 +- packages/google-compute-engine/packaging/setup_rpm.sh | 2 +- .../packaging/debian/changelog | 7 +++++++ .../python-google-compute-engine/packaging/setup_deb.sh | 2 +- .../python-google-compute-engine/packaging/setup_rpm.sh | 2 +- packages/python-google-compute-engine/setup.py | 2 +- 11 files changed, 28 insertions(+), 8 deletions(-) diff --git a/packages/google-compute-engine-oslogin/packaging/debian/changelog b/packages/google-compute-engine-oslogin/packaging/debian/changelog index bb4a1b54..5eb8a46a 100644 --- a/packages/google-compute-engine-oslogin/packaging/debian/changelog +++ b/packages/google-compute-engine-oslogin/packaging/debian/changelog @@ -1,3 +1,9 @@ +google-compute-engine-oslogin (1:20190801.00-g1) stable; urgency=medium + + * Correct JSON refcount decrementing. + + -- Google Cloud Team Thu, 01 Aug 2019 13:57:16 -0700 + google-compute-engine-oslogin (1:20190729.00-g1) stable; urgency=medium * Remove unnecessary binary search logic. diff --git a/packages/google-compute-engine-oslogin/packaging/setup_deb.sh b/packages/google-compute-engine-oslogin/packaging/setup_deb.sh index d9de78c0..5b3f8931 100755 --- a/packages/google-compute-engine-oslogin/packaging/setup_deb.sh +++ b/packages/google-compute-engine-oslogin/packaging/setup_deb.sh @@ -14,7 +14,7 @@ # limitations under the License. NAME="google-compute-engine-oslogin" -VERSION="20190729.00" +VERSION="20190801.00" DEB=$(cut -d. -f1 Thu, 01 Aug 2019 14:06:02 -0700 + google-compute-engine (1:20190729.00-g1) stable; urgency=medium * Suport Google Private Access over IPv6. diff --git a/packages/google-compute-engine/packaging/setup_deb.sh b/packages/google-compute-engine/packaging/setup_deb.sh index d4c84e9c..42ef2667 100755 --- a/packages/google-compute-engine/packaging/setup_deb.sh +++ b/packages/google-compute-engine/packaging/setup_deb.sh @@ -14,7 +14,7 @@ # limitations under the License. NAME="google-compute-engine" -VERSION="20190729.00" +VERSION="20190801.00" working_dir=${PWD} if [[ $(basename "$working_dir") != $NAME ]]; then diff --git a/packages/google-compute-engine/packaging/setup_rpm.sh b/packages/google-compute-engine/packaging/setup_rpm.sh index 119caaab..42e9c15f 100755 --- a/packages/google-compute-engine/packaging/setup_rpm.sh +++ b/packages/google-compute-engine/packaging/setup_rpm.sh @@ -14,7 +14,7 @@ # limitations under the License. NAME="google-compute-engine" -VERSION="20190729.00" +VERSION="20190801.00" rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION} working_dir=${PWD} diff --git a/packages/python-google-compute-engine/packaging/debian/changelog b/packages/python-google-compute-engine/packaging/debian/changelog index e919d55c..d1696408 100644 --- a/packages/python-google-compute-engine/packaging/debian/changelog +++ b/packages/python-google-compute-engine/packaging/debian/changelog @@ -1,3 +1,10 @@ +python-google-compute-engine (1:20190801.00-g1) stable; urgency=medium + + * Re-enable boto config and drop writing plugin directory. + * Fix metadata script retrieval. + + -- Google Cloud Team Thu, 01 Aug 2019 13:55:46 -0700 + python-google-compute-engine (1:20190729.00-g1) stable; urgency=medium * Support for Google Private Access over IPv6. diff --git a/packages/python-google-compute-engine/packaging/setup_deb.sh b/packages/python-google-compute-engine/packaging/setup_deb.sh index 573dba4a..9dc83065 100755 --- a/packages/python-google-compute-engine/packaging/setup_deb.sh +++ b/packages/python-google-compute-engine/packaging/setup_deb.sh @@ -14,7 +14,7 @@ # limitations under the License. NAME="python-google-compute-engine" -VERSION="20190729.00" +VERSION="20190801.00" working_dir=${PWD} if [[ $(basename "$working_dir") != $NAME ]]; then diff --git a/packages/python-google-compute-engine/packaging/setup_rpm.sh b/packages/python-google-compute-engine/packaging/setup_rpm.sh index e12be10f..f9266734 100755 --- a/packages/python-google-compute-engine/packaging/setup_rpm.sh +++ b/packages/python-google-compute-engine/packaging/setup_rpm.sh @@ -14,7 +14,7 @@ # limitations under the License. NAME="python-google-compute-engine" -VERSION="20190729.00" +VERSION="20190801.00" rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION} working_dir=${PWD} diff --git a/packages/python-google-compute-engine/setup.py b/packages/python-google-compute-engine/setup.py index a4fee36d..c51b5ffb 100755 --- a/packages/python-google-compute-engine/setup.py +++ b/packages/python-google-compute-engine/setup.py @@ -37,7 +37,7 @@ name='google-compute-engine', packages=setuptools.find_packages(), url='https://github.com/GoogleCloudPlatform/compute-image-packages', - version='20190729.00', + version='20190801.00', # Entry points create scripts in /usr/bin that call a function. entry_points={ 'console_scripts': [ From 17e2c6eb0bc5c073bbe1ca1a97f0c32daedc9c7c Mon Sep 17 00:00:00 2001 From: Matt Houglum Date: Thu, 1 Aug 2019 15:03:44 -0700 Subject: [PATCH 5/5] Add msg to consult gsutil team before changing cfg (#826) * Add msg to consult gsutil team before changing cfg This should help prevent issues like #820. Fixes #820. (Note that #823 added the fix; this simply closes out an additional action-item resulting from that issue.) * Add another msg where we decide to write the cfg --- .../google_compute_engine/boto/boto_config.py | 6 ++++++ .../google_compute_engine/instance_setup/instance_config.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/packages/python-google-compute-engine/google_compute_engine/boto/boto_config.py b/packages/python-google-compute-engine/google_compute_engine/boto/boto_config.py index 155f3c7e..2c92d9df 100644 --- a/packages/python-google-compute-engine/google_compute_engine/boto/boto_config.py +++ b/packages/python-google-compute-engine/google_compute_engine/boto/boto_config.py @@ -35,6 +35,10 @@ class BotoConfig(object): """Creates a boto config file for standalone GSUtil.""" + # WARNING: The path should remain as /etc/boto.cfg in order to not break + # tools, such as gsutil, that rely on loading well-known Boto config paths. + # If you want to change this, please consult the gsutil team + # (GoogleCloudPlatform/gsutil) first. boto_config = constants.BOTOCONFDIR + '/etc/boto.cfg' boto_config_template = constants.BOTOCONFDIR + '/etc/boto.cfg.template' boto_config_script = os.path.abspath(__file__) @@ -81,6 +85,8 @@ def _CreateConfig(self, project_id): config_file=self.boto_config_template, config_header=self.boto_config_header) + # WARNING: If you want to change the contents of this config file, please + # consult the gsutil team (GoogleCloudPlatform/gsutil) first. config.SetOption('GSUtil', 'default_project_id', project_id) config.SetOption('GSUtil', 'default_api_version', '2') config.SetOption('GoogleCompute', 'service_account', 'default') diff --git a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py index 1c23171d..8f2fadf1 100644 --- a/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py +++ b/packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_config.py @@ -78,6 +78,8 @@ class InstanceConfig(config_manager.ConfigManager): 'host_key_types': 'ecdsa,ed25519,rsa', 'optimize_local_ssd': 'true', 'network_enabled': 'true', + # WARNING: Do not change the value of 'set_boto_config' without first + # consulting the gsutil team (GoogleCloudPlatform/gsutil). 'set_boto_config': 'true', 'set_host_keys': 'true', 'set_multiqueue': 'true',