# Copyright 2015 The Chromium Authors. All rights reserved.
|
# Use of this source code is governed by a BSD-style license that can be
|
# found in the LICENSE file.
|
|
import json
|
import logging
|
import os
|
|
from py_utils import cloud_storage
|
from dependency_manager import archive_info
|
from dependency_manager import cloud_storage_info
|
from dependency_manager import dependency_info
|
from dependency_manager import exceptions
|
from dependency_manager import local_path_info
|
from dependency_manager import uploader
|
|
|
class BaseConfig(object):
|
"""A basic config class for use with the DependencyManager.
|
|
Initiated with a json file in the following format:
|
|
{ "config_type": "BaseConfig",
|
"dependencies": {
|
"dep_name1": {
|
"cloud_storage_base_folder": "base_folder1",
|
"cloud_storage_bucket": "bucket1",
|
"file_info": {
|
"platform1": {
|
"cloud_storage_hash": "hash_for_platform1",
|
"download_path": "download_path111",
|
"version_in_cs": "1.11.1.11."
|
"local_paths": ["local_path1110", "local_path1111"]
|
},
|
"platform2": {
|
"cloud_storage_hash": "hash_for_platform2",
|
"download_path": "download_path2",
|
"local_paths": ["local_path20", "local_path21"]
|
},
|
...
|
}
|
},
|
"dependency_name_2": {
|
...
|
},
|
...
|
}
|
}
|
|
Required fields: "dependencies" and "config_type".
|
Note that config_type must be "BaseConfig"
|
|
Assumptions:
|
"cloud_storage_base_folder" is a top level folder in the given
|
"cloud_storage_bucket" where all of the dependency files are stored
|
at "dependency_name"_"cloud_storage_hash".
|
|
"download_path" and all paths in "local_paths" are relative to the
|
config file's location.
|
|
All or none of the following cloud storage related fields must be
|
included in each platform dictionary:
|
"cloud_storage_hash", "download_path", "cs_remote_path"
|
|
"version_in_cs" is an optional cloud storage field, but is dependent
|
on the above cloud storage related fields.
|
|
|
Also note that platform names are often of the form os_architechture.
|
Ex: "win_AMD64"
|
|
More information on the fields can be found in dependencies_info.py
|
"""
|
def __init__(self, file_path, writable=False):
|
""" Initialize a BaseConfig for the DependencyManager.
|
|
Args:
|
writable: False: This config will be used to lookup information.
|
True: This config will be used to update information.
|
|
file_path: Path to a file containing a json dictionary in the expected
|
json format for this config class. Base format expected:
|
|
{ "config_type": config_type,
|
"dependencies": dependencies_dict }
|
|
config_type: must match the return value of GetConfigType.
|
dependencies: A dictionary with the information needed to
|
create dependency_info instances for the given
|
dependencies.
|
|
See dependency_info.py for more information.
|
"""
|
self._config_path = file_path
|
self._writable = writable
|
self._pending_uploads = []
|
if not self._config_path:
|
raise ValueError('Must supply config file path.')
|
if not os.path.exists(self._config_path):
|
if not writable:
|
raise exceptions.EmptyConfigError(file_path)
|
self._config_data = {}
|
self._WriteConfigToFile(self._config_path, dependencies=self._config_data)
|
else:
|
with open(file_path, 'r') as f:
|
config_data = json.load(f)
|
if not config_data:
|
raise exceptions.EmptyConfigError(file_path)
|
config_type = config_data.pop('config_type', None)
|
if config_type != self.GetConfigType():
|
raise ValueError(
|
'Supplied config_type (%s) is not the expected type (%s) in file '
|
'%s' % (config_type, self.GetConfigType(), file_path))
|
self._config_data = config_data.get('dependencies', {})
|
|
def IterDependencyInfo(self):
|
""" Yields a DependencyInfo for each dependency/platform pair.
|
|
Raises:
|
ReadWriteError: If called when the config is writable.
|
ValueError: If any of the dependencies contain partial information for
|
downloading from cloud_storage. (See dependency_info.py)
|
"""
|
if self._writable:
|
raise exceptions.ReadWriteError(
|
'Trying to read dependency info from a writable config. File for '
|
'config: %s' % self._config_path)
|
base_path = os.path.dirname(self._config_path)
|
for dependency in self._config_data:
|
dependency_dict = self._config_data.get(dependency)
|
platforms_dict = dependency_dict.get('file_info', {})
|
for platform in platforms_dict:
|
platform_info = platforms_dict.get(platform)
|
|
local_info = None
|
local_paths = platform_info.get('local_paths', [])
|
if local_paths:
|
paths = []
|
for path in local_paths:
|
path = self._FormatPath(path)
|
paths.append(os.path.abspath(os.path.join(base_path, path)))
|
local_info = local_path_info.LocalPathInfo(paths)
|
|
cs_info = None
|
cs_bucket = dependency_dict.get('cloud_storage_bucket')
|
cs_base_folder = dependency_dict.get('cloud_storage_base_folder', '')
|
download_path = platform_info.get('download_path')
|
if download_path:
|
download_path = self._FormatPath(download_path)
|
download_path = os.path.abspath(
|
os.path.join(base_path, download_path))
|
|
cs_hash = platform_info.get('cloud_storage_hash')
|
if not cs_hash:
|
raise exceptions.ConfigError(
|
'Dependency %s has cloud storage info on platform %s, but is '
|
'missing a cloud storage hash.', dependency, platform)
|
cs_remote_path = self._CloudStorageRemotePath(
|
dependency, cs_hash, cs_base_folder)
|
version_in_cs = platform_info.get('version_in_cs')
|
|
zip_info = None
|
path_within_archive = platform_info.get('path_within_archive')
|
if path_within_archive:
|
unzip_path = os.path.abspath(
|
os.path.join(os.path.dirname(download_path),
|
'%s_%s_%s' % (dependency, platform, cs_hash)))
|
stale_unzip_path_glob = os.path.abspath(
|
os.path.join(os.path.dirname(download_path),
|
'%s_%s_%s' % (dependency, platform,
|
'[0-9a-f]' * 40)))
|
zip_info = archive_info.ArchiveInfo(
|
download_path, unzip_path, path_within_archive,
|
stale_unzip_path_glob)
|
|
cs_info = cloud_storage_info.CloudStorageInfo(
|
cs_bucket, cs_hash, download_path, cs_remote_path,
|
version_in_cs=version_in_cs, archive_info=zip_info)
|
|
dep_info = dependency_info.DependencyInfo(
|
dependency, platform, self._config_path,
|
local_path_info=local_info, cloud_storage_info=cs_info)
|
yield dep_info
|
|
@classmethod
|
def GetConfigType(cls):
|
return 'BaseConfig'
|
|
@property
|
def config_path(self):
|
return self._config_path
|
|
def AddNewDependency(
|
self, dependency, cloud_storage_base_folder, cloud_storage_bucket):
|
self._ValidateIsConfigWritable()
|
if dependency in self:
|
raise ValueError('Config already contains dependency %s' % dependency)
|
self._config_data[dependency] = {
|
'cloud_storage_base_folder': cloud_storage_base_folder,
|
'cloud_storage_bucket': cloud_storage_bucket,
|
'file_info': {},
|
}
|
|
def SetDownloadPath(self, dependency, platform, download_path):
|
self._ValidateIsConfigWritable()
|
if not dependency in self:
|
raise ValueError('Config does not contain dependency %s' % dependency)
|
platform_dicts = self._config_data[dependency]['file_info']
|
if platform not in platform_dicts:
|
platform_dicts[platform] = {}
|
platform_dicts[platform]['download_path'] = download_path
|
|
def AddCloudStorageDependencyUpdateJob(
|
self, dependency, platform, dependency_path, version=None,
|
execute_job=True):
|
"""Update the file downloaded from cloud storage for a dependency/platform.
|
|
Upload a new file to cloud storage for the given dependency and platform
|
pair and update the cloud storage hash and the version for the given pair.
|
|
Example usage:
|
The following should update the default platform for 'dep_name':
|
UpdateCloudStorageDependency('dep_name', 'default', 'path/to/file')
|
|
The following should update both the mac and win platforms for 'dep_name',
|
or neither if either update fails:
|
UpdateCloudStorageDependency(
|
'dep_name', 'mac_x86_64', 'path/to/mac/file', execute_job=False)
|
UpdateCloudStorageDependency(
|
'dep_name', 'win_AMD64', 'path/to/win/file', execute_job=False)
|
ExecuteUpdateJobs()
|
|
Args:
|
dependency: The dependency to update.
|
platform: The platform to update the dependency info for.
|
dependency_path: Path to the new dependency to be used.
|
version: Version of the updated dependency, for checking future updates
|
against.
|
execute_job: True if the config should be written to disk and the file
|
should be uploaded to cloud storage after the update. False if
|
multiple updates should be performed atomically. Must call
|
ExecuteUpdateJobs after all non-executed jobs are added to complete
|
the update.
|
|
Raises:
|
ReadWriteError: If the config was not initialized as writable, or if
|
|execute_job| is True but the config has update jobs still pending
|
execution.
|
ValueError: If no information exists in the config for |dependency| on
|
|platform|.
|
"""
|
self._ValidateIsConfigUpdatable(
|
execute_job=execute_job, dependency=dependency, platform=platform)
|
cs_hash = cloud_storage.CalculateHash(dependency_path)
|
if version:
|
self._SetPlatformData(dependency, platform, 'version_in_cs', version)
|
self._SetPlatformData(dependency, platform, 'cloud_storage_hash', cs_hash)
|
|
cs_base_folder = self._GetPlatformData(
|
dependency, platform, 'cloud_storage_base_folder')
|
cs_bucket = self._GetPlatformData(
|
dependency, platform, 'cloud_storage_bucket')
|
cs_remote_path = self._CloudStorageRemotePath(
|
dependency, cs_hash, cs_base_folder)
|
self._pending_uploads.append(uploader.CloudStorageUploader(
|
cs_bucket, cs_remote_path, dependency_path))
|
if execute_job:
|
self.ExecuteUpdateJobs()
|
|
def ExecuteUpdateJobs(self, force=False):
|
"""Write all config changes to the config_path specified in __init__.
|
|
Upload all files pending upload and then write the updated config to
|
file. Attempt to remove all uploaded files on failure.
|
|
Args:
|
force: True if files should be uploaded to cloud storage even if a
|
file already exists in the upload location.
|
|
Returns:
|
True: if the config was dirty and the upload succeeded.
|
False: if the config was not dirty.
|
|
Raises:
|
CloudStorageUploadConflictError: If |force| is False and the potential
|
upload location of a file already exists.
|
CloudStorageError: If copying an existing file to the backup location
|
or uploading a new file fails.
|
"""
|
self._ValidateIsConfigUpdatable()
|
if not self._IsDirty():
|
logging.info('ExecuteUpdateJobs called on clean config')
|
return False
|
if not self._pending_uploads:
|
logging.debug('No files needing upload.')
|
else:
|
try:
|
for item_pending_upload in self._pending_uploads:
|
item_pending_upload.Upload(force)
|
self._WriteConfigToFile(self._config_path, self._config_data)
|
self._pending_uploads = []
|
except:
|
# Attempt to rollback the update in any instance of failure, even user
|
# interrupt via Ctrl+C; but don't consume the exception.
|
logging.error('Update failed, attempting to roll it back.')
|
for upload_item in reversed(self._pending_uploads):
|
upload_item.Rollback()
|
raise
|
return True
|
|
def GetVersion(self, dependency, platform):
|
"""Return the Version information for the given dependency."""
|
return self._GetPlatformData(
|
dependency, platform, data_type='version_in_cs')
|
|
def __contains__(self, dependency):
|
""" Returns whether this config contains |dependency|
|
|
Args:
|
dependency: the string name of dependency
|
"""
|
return dependency in self._config_data
|
|
def _IsDirty(self):
|
with open(self._config_path, 'r') as fstream:
|
curr_config_data = json.load(fstream)
|
curr_config_data = curr_config_data.get('dependencies', {})
|
return self._config_data != curr_config_data
|
|
def _SetPlatformData(self, dependency, platform, data_type, data):
|
self._ValidateIsConfigWritable()
|
dependency_dict = self._config_data.get(dependency, {})
|
platform_dict = dependency_dict.get('file_info', {}).get(platform)
|
if not platform_dict:
|
raise ValueError('No platform data for platform %s on dependency %s' %
|
(platform, dependency))
|
if (data_type == 'cloud_storage_bucket' or
|
data_type == 'cloud_storage_base_folder'):
|
self._config_data[dependency][data_type] = data
|
else:
|
self._config_data[dependency]['file_info'][platform][data_type] = data
|
|
def _GetPlatformData(self, dependency, platform, data_type=None):
|
dependency_dict = self._config_data.get(dependency, {})
|
if not dependency_dict:
|
raise ValueError('Dependency %s is not in config.' % dependency)
|
platform_dict = dependency_dict.get('file_info', {}).get(platform)
|
if not platform_dict:
|
raise ValueError('No platform data for platform %s on dependency %s' %
|
(platform, dependency))
|
if data_type:
|
if (data_type == 'cloud_storage_bucket' or
|
data_type == 'cloud_storage_base_folder'):
|
return dependency_dict.get(data_type)
|
return platform_dict.get(data_type)
|
return platform_dict
|
|
def _ValidateIsConfigUpdatable(
|
self, execute_job=False, dependency=None, platform=None):
|
self._ValidateIsConfigWritable()
|
if self._IsDirty() and execute_job:
|
raise exceptions.ReadWriteError(
|
'A change has already been made to this config. Either call without'
|
'using the execute_job option or first call ExecuteUpdateJobs().')
|
if dependency and not self._config_data.get(dependency):
|
raise ValueError('Cannot update information because dependency %s does '
|
'not exist.' % dependency)
|
if platform and not self._GetPlatformData(dependency, platform):
|
raise ValueError('No dependency info is available for the given '
|
'dependency: %s' % dependency)
|
|
def _ValidateIsConfigWritable(self):
|
if not self._writable:
|
raise exceptions.ReadWriteError(
|
'Trying to update the information from a read-only config. '
|
'File for config: %s' % self._config_path)
|
|
@staticmethod
|
def _CloudStorageRemotePath(dependency, cs_hash, cs_base_folder):
|
cs_remote_file = '%s_%s' % (dependency, cs_hash)
|
cs_remote_path = cs_remote_file if not cs_base_folder else (
|
'%s/%s' % (cs_base_folder, cs_remote_file))
|
return cs_remote_path
|
|
@classmethod
|
def _FormatPath(cls, file_path):
|
""" Format |file_path| for the current file system.
|
|
We may be downloading files for another platform, so paths must be
|
downloadable on the current system.
|
"""
|
if not file_path:
|
return file_path
|
if os.path.sep != '\\':
|
return file_path.replace('\\', os.path.sep)
|
elif os.path.sep != '/':
|
return file_path.replace('/', os.path.sep)
|
return file_path
|
|
@classmethod
|
def _WriteConfigToFile(cls, file_path, dependencies=None):
|
json_dict = cls._GetJsonDict(dependencies)
|
file_dir = os.path.dirname(file_path)
|
if not os.path.exists(file_dir):
|
os.makedirs(file_dir)
|
with open(file_path, 'w') as outfile:
|
json.dump(
|
json_dict, outfile, indent=2, sort_keys=True, separators=(',', ': '))
|
return json_dict
|
|
@classmethod
|
def _GetJsonDict(cls, dependencies=None):
|
dependencies = dependencies or {}
|
json_dict = {'config_type': cls.GetConfigType(),
|
'dependencies': dependencies}
|
return json_dict
|
