Merge branch 'development'

daisy_workflows/build_deb_packages.sh

@@ -33,7 +33,8 @@ fi
 workdir=$(pwd)
 mkdir output
 
-sudo apt-get install -y git
+apt-get update
+apt-get install -y git
 
 # Clone the github repo.
 git clone ${GIT_REPO} -b ${BRANCH} compute-image-packages

packages/google-compute-engine/packaging/debian/changelog

@@ -1,3 +1,9 @@
+google-compute-engine (1:20190916.00-g2) stable; urgency=medium
+
+  * Update dependencies.
+
+ -- Google Cloud Team <gc-team@google.com>  Mon, 16 Sep 2019 15:51:24 -0700
+
 google-compute-engine (1:20190916.00-g1) stable; urgency=medium
 
   * Bump version to match python package release.

packages/google-compute-engine/packaging/google-compute-engine-el6.spec

@@ -15,14 +15,14 @@
 Name: google-compute-engine
 Epoch: 1
 Version: %{_version}
-Release: g1.el6
+Release: g2.el6
 Summary: Google Compute Engine guest environment.
 License: ASL 2.0
 Url: https://github.com/GoogleCloudPlatform/compute-image-packages
 Source0: %{name}_%{version}.orig.tar.gz
 Requires: curl
 Requires: google-compute-engine-oslogin
-Requires: python-google-compute-engine = 1:%{version}
+Requires: python-google-compute-engine >= 1:20190916.00
 Requires: rsyslog
 # Old packages.
 Obsoletes: google-compute-engine-init

packages/google-compute-engine/packaging/google-compute-engine.spec

@@ -20,17 +20,17 @@
 Name: google-compute-engine
 Epoch: 1
 Version: %{_version}
-Release: g1%{?dist}
+Release: g2%{?dist}
 Summary: Google Compute Engine guest environment.
 License: ASL 2.0
 Url: https://github.com/GoogleCloudPlatform/compute-image-packages
 Source0: %{name}_%{version}.orig.tar.gz
 Requires: curl
 Requires: google-compute-engine-oslogin
 %if 0%{?rhel} == 8
-Requires: python3-google-compute-engine = 1:%{version}
+Requires: python3-google-compute-engine >= 1:20190916.00
 %else
-Requires: python-google-compute-engine = 1:%{version}
+Requires: python-google-compute-engine >= 1:20190916.00
 %endif
 Requires: rsyslog
 

packages/python-google-compute-engine/google_compute_engine/distro_lib/helpers.py

@@ -17,6 +17,7 @@
 
 import os
 import subprocess
+import time
 
 
 def CallDhclient(
@@ -68,6 +69,23 @@ def CallDhclientIpv6(interfaces, logger, dhclient_script=None,
                      interfaces)
     return
 
+  # Check for a 'tentative' IPv6 address which would prevent `dhclient -6` from
+  # succeeding below. This should only take 1 second, but we try for up to 5.
+  command = ['ip', '-6', '-o', 'a', 's', 'dev', interfaces[0], 'scope',
+             'link', 'tentative']
+  for i in range(5):
+    output = ''
+    try:
+      output = subprocess.check_output(command)
+    except subprocess.CalledProcessError as e:
+      logger.warning('Could not confirm tentative IPv6 address: %s.', e.output)
+    if output:
+      logger.info('Found tentative ipv6 link address %s, sleeping 1 second.',
+                  output.strip())
+      time.sleep(1)
+    else:
+      break
+
   if dhclient_script and os.path.exists(dhclient_script):
     dhclient_command += ['-sf', dhclient_script]
 

packages/python-google-compute-engine/google_compute_engine/instance_setup/instance_setup.py

@@ -28,7 +28,7 @@
 from google_compute_engine import logger
 from google_compute_engine import metadata_watcher
 from google_compute_engine.boto import boto_config
-from google_compute_engine.compat import urlerror
+from google_compute_engine.compat import distro_name, urlerror
 from google_compute_engine.compat import urlrequest
 from google_compute_engine.instance_setup import instance_config
 
@@ -65,17 +65,27 @@ def __init__(self, debug=False):
       instance_config_metadata = self._GetInstanceConfig()
       self.instance_config = instance_config.InstanceConfig(
           logger=self.logger, instance_config_metadata=instance_config_metadata)
+
       if self.instance_config.GetOptionBool('InstanceSetup', 'set_host_keys'):
         host_key_types = self.instance_config.GetOptionString(
             'InstanceSetup', 'host_key_types')
         self._SetSshHostKeys(host_key_types=host_key_types)
+
       if self.instance_config.GetOptionBool('InstanceSetup', 'set_boto_config'):
         self._SetupBotoConfig()
+
+      # machineType is e.g. u'projects/00000000000000/machineTypes/n1-standard-1'
+      machineType = self.metadata_dict['instance']['machineType'].split('/')[-1]
+      if machineType.startswith("e2-") and 'bsd' not in distro_name:  # Not yet supported on BSD.
+        subprocess.call(["sysctl", "vm.overcommit_memory=1"])
+
     if self.instance_config.GetOptionBool(
         'InstanceSetup', 'optimize_local_ssd'):
       self._RunScript('google_optimize_local_ssd')
+
     if self.instance_config.GetOptionBool('InstanceSetup', 'set_multiqueue'):
       self._RunScript('google_set_multiqueue')
+
     try:
       self.instance_config.WriteConfig()
     except (IOError, OSError) as e:
@@ -205,10 +215,9 @@ def _SetSshHostKeys(self, host_key_types=None):
     Args:
       host_key_types: string, a comma separated list of host key types.
     """
-    section = 'Instance'
     instance_id = self._GetInstanceId()
     if instance_id != self.instance_config.GetOptionString(
-        section, 'instance_id'):
+        'instance', 'instance_id'):
       self.logger.info('Generating SSH host keys for instance %s.', instance_id)
       file_regex = re.compile(r'ssh_host_(?P<type>[a-z0-9]*)_key\Z')
       key_dir = '/etc/ssh'
@@ -222,7 +231,7 @@ def _SetSshHostKeys(self, host_key_types=None):
         if key_data:
           self._WriteHostKeyToGuestAttributes(key_data[0], key_data[1])
       self._StartSshd()
-      self.instance_config.SetOption(section, 'instance_id', str(instance_id))
+      self.instance_config.SetOption('Instance', 'instance_id', str(instance_id))
 
   def _GetNumericProjectId(self):
     """Get the numeric project ID.

packages/python-google-compute-engine/google_compute_engine/instance_setup/tests/instance_setup_test.py

@@ -46,7 +46,7 @@ def testInstanceSetup(self, mock_logger, mock_watcher, mock_config):
     mock_logger_instance = mock.Mock()
     mock_logger.Logger.return_value = mock_logger_instance
     mock_watcher_instance = mock.Mock()
-    mock_watcher_instance.GetMetadata.return_value = {'hello': 'world'}
+    mock_watcher_instance.GetMetadata.return_value = {'hello': 'world', 'instance': {'machineType': 'fake'}}
     mock_watcher.MetadataWatcher.return_value = mock_watcher_instance
     mock_config_instance = mock.Mock()
     mock_config_instance.GetOptionBool.return_value = True
@@ -91,7 +91,7 @@ def testInstanceSetup(self, mock_logger, mock_watcher, mock_config):
         mock.call.config.InstanceConfig().WriteConfig(),
     ]
     self.assertEqual(mocks.mock_calls, expected_calls)
-    self.assertEqual(mock_setup.metadata_dict, {'hello': 'world'})
+    self.assertEqual(mock_setup.metadata_dict, {'hello': 'world', 'instance': {'machineType': 'fake'}})
 
   @mock.patch('google_compute_engine.instance_setup.instance_setup.instance_config')
   @mock.patch('google_compute_engine.instance_setup.instance_setup.metadata_watcher')
@@ -386,7 +386,7 @@ def testSetSshHostKeysFirstBoot(self, mock_listdir):
     ]
 
     self.assertEqual(sorted(mock_generate_key.mock_calls), expected_calls)
-    self.mock_instance_config.SetOption.assert_called_once_with(
+    self.mock_instance_config.SetOption.assert_called_with(
         'Instance', 'instance_id', '123')
 
   def testGetNumericProjectId(self):

packages/python-google-compute-engine/google_compute_engine/metadata_scripts/script_retriever.py

@@ -97,7 +97,7 @@ def _DownloadAuthUrl(self, url, dest_dir):
 
     if not self.token:
       response = self.watcher.GetMetadata(
-          self.token_metadata_key, recursive=False, retry=False)
+          self.token_metadata_key, recursive=False, retries=1)
 
       if not response:
         self.logger.info(

packages/python-google-compute-engine/google_compute_engine/metadata_scripts/tests/script_retriever_test.py

@@ -89,7 +89,7 @@ def testDownloadAuthUrlExceptionAndToken(
     # GetMetadata includes a prefix, so remove it.
     stripped_url = token_url.replace(metadata_prefix, '')
     mock_get_metadata.assert_called_once_with(
-        stripped_url, recursive=False, retry=False)
+        stripped_url, recursive=False, retries=1)
 
     self.assertEqual(self.retriever.token, 'foo bar')
 
@@ -119,7 +119,7 @@ def testDownloadAuthUrlFallback(
     prefix = 'http://metadata.google.internal/computeMetadata/v1/'
     stripped_url = token_url.replace(prefix, '')
     mock_get_metadata.assert_called_once_with(
-        stripped_url, recursive=False, retry=False)
+        stripped_url, recursive=False, retries=1)
     mock_download_url.assert_called_once_with(auth_url, self.dest_dir)
 
     self.assertIsNone(self.retriever.token)

packages/python-google-compute-engine/google_compute_engine/metadata_watcher.py

@@ -154,33 +154,31 @@ def _GetMetadataUpdate(
 
   def _HandleMetadataUpdate(
       self, metadata_key='', recursive=True, wait=True, timeout=None,
-      retry=True):
+      retries=5):
     """Wait for a successful metadata response.
 
     Args:
       metadata_key: string, the metadata key to watch for changes.
       recursive: bool, True if we should recursively watch for metadata changes.
       wait: bool, True if we should wait for a metadata change.
       timeout: int, timeout in seconds for returning metadata output.
-      retry: bool, True if we should retry on failure.
+      retries: int, Number of times to retry obtaining metadata.
 
     Returns:
       json, the deserialized contents of the metadata server.
     """
     exception = None
-    while True:
+    while retries >= 0:
       try:
         return self._GetMetadataUpdate(
             metadata_key=metadata_key, recursive=recursive, wait=wait,
             timeout=timeout)
       except (httpclient.HTTPException, socket.error, urlerror.URLError) as e:
+        retries -= 1
         if not isinstance(e, type(exception)):
           exception = e
           self.logger.error('GET request error retrieving metadata. %s.', e)
-        if retry:
-          continue
-        else:
-          break
+        time.sleep(1)
 
   def WatchMetadata(
       self, handler, metadata_key='', recursive=True, timeout=None):
@@ -202,7 +200,7 @@ def WatchMetadata(
         self.logger.exception('Exception calling the response handler. %s.', e)
 
   def GetMetadata(
-      self, metadata_key='', recursive=True, timeout=None, retry=True):
+      self, metadata_key='', recursive=True, timeout=None, retries=5):
     """Retrieve the contents of metadata server for a metadata key.
 
     Args:
@@ -216,4 +214,4 @@ def GetMetadata(
     """
     return self._HandleMetadataUpdate(
         metadata_key=metadata_key, recursive=recursive, wait=False,
-        timeout=timeout, retry=retry)
+        timeout=timeout, retries=retries)

packages/python-google-compute-engine/google_compute_engine/tests/metadata_watcher_test.py

@@ -259,12 +259,12 @@ def testHandleMetadataUpdateException(self):
     metadata_key = 'instance/id'
     recursive = False
     wait = False
-    retry = True
+    retries = 5
 
     self.assertEqual(
         self.mock_watcher._HandleMetadataUpdate(
             metadata_key=metadata_key, recursive=recursive, wait=wait,
-            timeout=None, retry=retry),
+            timeout=None, retries=retries),
         {})
     expected_calls = [
         mock.call(
@@ -282,12 +282,12 @@ def testHandleMetadataUpdateExceptionNoRetry(self):
     metadata_key = 'instance/id'
     recursive = False
     wait = False
-    retry = False
+    retries = 0
 
     self.assertIsNone(
         self.mock_watcher._HandleMetadataUpdate(
             metadata_key=metadata_key, recursive=recursive, wait=wait,
-            timeout=None, retry=retry))
+            timeout=None, retries=retries))
     expected_calls = [
         mock.call(
             metadata_key=metadata_key, recursive=recursive, wait=wait,
@@ -333,7 +333,7 @@ def testGetMetadata(self):
 
     self.assertEqual(self.mock_watcher.GetMetadata(), {})
     mock_response.assert_called_once_with(
-        metadata_key='', recursive=True, wait=False, timeout=None, retry=True)
+        metadata_key='', recursive=True, wait=False, timeout=None, retries=5)
     self.mock_watcher.logger.exception.assert_not_called()
 
   def testGetMetadataArgs(self):
@@ -342,15 +342,15 @@ def testGetMetadataArgs(self):
     self.mock_watcher._HandleMetadataUpdate = mock_response
     metadata_key = 'instance/id'
     recursive = False
-    retry = False
+    retries = 5
 
     response = self.mock_watcher.GetMetadata(
         metadata_key=metadata_key, recursive=recursive, timeout=60,
-        retry=retry)
+        retries=retries)
     self.assertEqual(response, {})
     mock_response.assert_called_once_with(
         metadata_key=metadata_key, recursive=False, wait=False, timeout=60,
-        retry=False)
+        retries=5)
     self.mock_watcher.logger.exception.assert_not_called()
 
 

packages/python-google-compute-engine/packaging/debian/changelog

@@ -1,3 +1,11 @@
+python-google-compute-engine (1:20191112.00-g1) stable; urgency=medium
+
+  * Enable sysctl change for E2 platform.
+  * Retry metadata lookups in agent.
+  * Wait for link-local IPv6 addresses to resolve.
+
+ -- Google Cloud Team <gc-team@google.com>  Mon, 16 Sep 2019 15:52:24 -0700
+
 python-google-compute-engine (1:20190916.00-g1) stable; urgency=medium
 
   * Revert VM rename feature.

packages/python-google-compute-engine/packaging/setup_deb.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 NAME="python-google-compute-engine"
-VERSION="20190916.00"
+VERSION="20191112.00"
 
 working_dir=${PWD}
 if [[ $(basename "$working_dir") != $NAME ]]; then

packages/python-google-compute-engine/packaging/setup_rpm.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 NAME="python-google-compute-engine"
-VERSION="20190916.00"
+VERSION="20191112.00"
 
 rpm_working_dir=/tmp/rpmpackage/${NAME}-${VERSION}
 working_dir=${PWD}

packages/python-google-compute-engine/setup.py

@@ -37,7 +37,7 @@
     name='google-compute-engine',
     packages=setuptools.find_packages(),
     url='https://github.com/GoogleCloudPlatform/compute-image-packages',
-    version='20190916.0',
+    version='20191112.0',
     # Entry points create scripts in /usr/bin that call a function.
     entry_points={
         'console_scripts': [