=== modified file 'Makefile' --- Makefile 2016-05-10 16:13:29 +0000 +++ Makefile 2016-10-03 18:55:20 +0000 @@ -49,5 +49,7 @@ sync-images: @$(CWD)/tools/vmtest-sync-images +clean: + rm -rf doc/_build -.PHONY: all test pyflakes pyflakes3 pep8 build +.PHONY: all clean test pyflakes pyflakes3 pep8 build === modified file 'curtin/__init__.py' --- curtin/__init__.py 2015-11-23 16:22:09 +0000 +++ curtin/__init__.py 2016-10-03 18:55:20 +0000 @@ -33,6 +33,10 @@ 'SUBCOMMAND_SYSTEM_INSTALL', # subcommand 'system-upgrade' is present 'SUBCOMMAND_SYSTEM_UPGRADE', + # supports new format of apt configuration + 'APT_CONFIG_V1', ] +__version__ = "0.1.0" + # vi: ts=4 expandtab syntax=python === modified file 'curtin/block/__init__.py' --- curtin/block/__init__.py 2016-10-03 18:00:41 +0000 +++ curtin/block/__init__.py 2016-10-03 18:55:20 +0000 @@ -23,21 +23,31 @@ import itertools from curtin import util +from curtin.block import lvm +from curtin.log import LOG from curtin.udev import udevadm_settle -from curtin.log import LOG def get_dev_name_entry(devname): + """ + convert device name to path in /dev + """ bname = devname.split('/dev/')[-1] return (bname, "/dev/" + bname) def is_valid_device(devname): + """ + check if device is a valid device + """ devent = get_dev_name_entry(devname)[1] return is_block_device(devent) def is_block_device(path): + """ + check if path is a block device + """ try: return stat.S_ISBLK(os.stat(path).st_mode) except OSError as e: @@ -47,26 +57,99 @@ def dev_short(devname): + """ + get short form of device name + """ + devname = os.path.normpath(devname) if os.path.sep in devname: return os.path.basename(devname) return devname def dev_path(devname): + """ + convert device name to path in /dev + """ if devname.startswith('/dev/'): return devname else: return '/dev/' + devname +def path_to_kname(path): + """ + converts a path in /dev or a path in /sys/block to the device kname, + taking special devices and unusual naming schemes into account + """ + # if path given is a link, get real path + # only do this if given a path though, if kname is already specified then + # this would cause a failure where the function should still be able to run + if os.path.sep in path: + path = os.path.realpath(path) + # using basename here ensures that the function will work given a path in + # /dev, a kname, or a path in /sys/block as an arg + dev_kname = os.path.basename(path) + # cciss devices need to have 'cciss!' prepended + if path.startswith('/dev/cciss'): + dev_kname = 'cciss!' + dev_kname + LOG.debug("path_to_kname input: '{}' output: '{}'".format(path, dev_kname)) + return dev_kname + + +def kname_to_path(kname): + """ + converts a kname to a path in /dev, taking special devices and unusual + naming schemes into account + """ + # if given something that is already a dev path, return it + if os.path.exists(kname) and is_valid_device(kname): + path = kname + LOG.debug("kname_to_path input: '{}' output: '{}'".format(kname, path)) + return os.path.realpath(path) + # adding '/dev' to path is not sufficient to handle cciss devices and + # possibly other special devices which have not been encountered yet + path = os.path.realpath(os.sep.join(['/dev'] + kname.split('!'))) + # make sure path we get is correct + if not (os.path.exists(path) and is_valid_device(path)): + raise OSError('could not get path to dev from kname: {}'.format(kname)) + LOG.debug("kname_to_path input: '{}' output: '{}'".format(kname, path)) + return path + + +def partition_kname(disk_kname, partition_number): + """ + Add number to disk_kname prepending a 'p' if needed + """ + for dev_type in ['nvme', 'mmcblk', 'cciss', 'mpath', 'dm']: + if disk_kname.startswith(dev_type): + partition_number = "p%s" % partition_number + break + return "%s%s" % (disk_kname, partition_number) + + +def sysfs_to_devpath(sysfs_path): + """ + convert a path in /sys/class/block to a path in /dev + """ + path = kname_to_path(path_to_kname(sysfs_path)) + if not is_block_device(path): + raise ValueError('could not find blockdev for sys path: {}' + .format(sysfs_path)) + return path + + def sys_block_path(devname, add=None, strict=True): + """ + get path to device in /sys/class/block + """ toks = ['/sys/class/block'] # insert parent dev if devname is partition + devname = os.path.normpath(devname) (parent, partnum) = get_blockdev_for_partition(devname) if partnum: - toks.append(dev_short(parent)) + toks.append(path_to_kname(parent)) - toks.append(dev_short(devname)) + toks.append(path_to_kname(devname)) if add is not None: toks.append(add) @@ -83,6 +166,9 @@ def _lsblock_pairs_to_dict(lines): + """ + parse lsblock output and convert to dict + """ ret = {} for line in lines.splitlines(): toks = shlex.split(line) @@ -98,6 +184,9 @@ def _lsblock(args=None): + """ + get lsblock data as dict + """ # lsblk --help | sed -n '/Available/,/^$/p' | # sed -e 1d -e '$d' -e 's,^[ ]\+,,' -e 's, .*,,' | sort keys = ['ALIGNMENT', 'DISC-ALN', 'DISC-GRAN', 'DISC-MAX', 'DISC-ZERO', @@ -120,8 +209,10 @@ def get_unused_blockdev_info(): - # return a list of unused block devices. These are devices that - # do not have anything mounted on them. + """ + return a list of unused block devices. + These are devices that do not have anything mounted on them. + """ # get a list of top level block devices, then iterate over it to get # devices dependent on those. If the lsblk call for that specific @@ -137,7 +228,9 @@ def get_devices_for_mp(mountpoint): - # return a list of devices (full paths) used by the provided mountpoint + """ + return a list of devices (full paths) used by the provided mountpoint + """ bdinfo = _lsblock() found = set() for devname, data in bdinfo.items(): @@ -158,6 +251,9 @@ def get_installable_blockdevs(include_removable=False, min_size=1024**3): + """ + find blockdevs suitable for installation + """ good = [] unused = get_unused_blockdev_info() for devname, data in unused.items(): @@ -172,21 +268,25 @@ def get_blockdev_for_partition(devpath): + """ + find the parent device for a partition. + returns a tuple of the parent block device and the partition number + if device is not a partition, None will be returned for partition number + """ + # normalize path + rpath = os.path.realpath(devpath) + # convert an entry in /dev/ to parent disk and partition number # if devpath is a block device and not a partition, return (devpath, None) - - # input of /dev/vdb or /dev/disk/by-label/foo - # rpath is hopefully a real-ish path in /dev (vda, sdb..) - rpath = os.path.realpath(devpath) - - bname = os.path.basename(rpath) - syspath = "/sys/class/block/%s" % bname - + base = '/sys/class/block' + + # input of /dev/vdb, /dev/disk/by-label/foo, /sys/block/foo, + # /sys/block/class/foo, or just foo + syspath = os.path.join(base, path_to_kname(devpath)) + + # don't need to try out multiple sysfs paths as path_to_kname handles cciss if not os.path.exists(syspath): - syspath2 = "/sys/class/block/cciss!%s" % bname - if not os.path.exists(syspath2): - raise ValueError("%s had no syspath (%s)" % (devpath, syspath)) - syspath = syspath2 + raise OSError("%s had no syspath (%s)" % (devpath, syspath)) ptpath = os.path.join(syspath, "partition") if not os.path.exists(ptpath): @@ -207,8 +307,21 @@ return (diskdevpath, ptnum) +def get_sysfs_partitions(device): + """ + get a list of sysfs paths for partitions under a block device + accepts input as a device kname, sysfs path, or dev path + returns empty list if no partitions available + """ + sysfs_path = sys_block_path(device) + return [sys_block_path(kname) for kname in os.listdir(sysfs_path) + if os.path.exists(os.path.join(sysfs_path, kname, 'partition'))] + + def get_pardevs_on_blockdevs(devs): - # return a dict of partitions with their info that are on provided devs + """ + return a dict of partitions with their info that are on provided devs + """ if devs is None: devs = [] devs = [get_dev_name_entry(d)[1] for d in devs] @@ -243,7 +356,9 @@ def rescan_block_devices(): - # run 'blockdev --rereadpt' for all block devices not currently mounted + """ + run 'blockdev --rereadpt' for all block devices not currently mounted + """ unused = get_unused_blockdev_info() devices = [] for devname, data in unused.items(): @@ -271,6 +386,9 @@ def blkid(devs=None, cache=True): + """ + get data about block devices from blkid and convert to dict + """ if devs is None: devs = [] @@ -423,7 +541,18 @@ """ info = _lsblock([devpath]) LOG.debug('get_blockdev_sector_size: info:\n%s' % util.json_dumps(info)) - [parent] = info + # (LP: 1598310) The call to _lsblock() may return multiple results. + # If it does, then search for a result with the correct device path. + # If no such device is found among the results, then fall back to previous + # behavior, which was taking the first of the results + assert len(info) > 0 + for (k, v) in info.items(): + if v.get('device_path') == devpath: + parent = k + break + else: + parent = list(info.keys())[0] + return (int(info[parent]['LOG-SEC']), int(info[parent]['PHY-SEC'])) @@ -499,50 +628,108 @@ def sysfs_partition_data(blockdev=None, sysfs_path=None): # given block device or sysfs_path, return a list of tuples # of (kernel_name, number, offset, size) - if blockdev is None and sysfs_path is None: - raise ValueError("Blockdev and sysfs_path cannot both be None") - if blockdev: + blockdev = os.path.normpath(blockdev) sysfs_path = sys_block_path(blockdev) - - ptdata = [] - # /sys/class/block/dev has entries of 'kname' for each partition + elif sysfs_path: + # use normpath to ensure that paths with trailing slash work + sysfs_path = os.path.normpath(sysfs_path) + blockdev = os.path.join('/dev', os.path.basename(sysfs_path)) + else: + raise ValueError("Blockdev and sysfs_path cannot both be None") # queue property is only on parent devices, ie, we can't read # /sys/class/block/vda/vda1/queue/* as queue is only on the # parent device + sysfs_prefix = sysfs_path (parent, partnum) = get_blockdev_for_partition(blockdev) - sysfs_prefix = sysfs_path if partnum: sysfs_prefix = sys_block_path(parent) - - block_size = int(util.load_file(os.path.join(sysfs_prefix, - 'queue/logical_block_size'))) - - block_size = int( - util.load_file(os.path.join(sysfs_path, 'queue/logical_block_size'))) + partnum = int(partnum) + + block_size = int(util.load_file(os.path.join( + sysfs_prefix, 'queue/logical_block_size'))) unit = block_size - for d in os.listdir(sysfs_path): - partd = os.path.join(sysfs_path, d) + + ptdata = [] + for part_sysfs in get_sysfs_partitions(sysfs_prefix): data = {} for sfile in ('partition', 'start', 'size'): - dfile = os.path.join(partd, sfile) + dfile = os.path.join(part_sysfs, sfile) if not os.path.isfile(dfile): continue data[sfile] = int(util.load_file(dfile)) - if 'partition' not in data: - continue - ptdata.append((d, data['partition'], data['start'] * unit, - data['size'] * unit,)) + if partnum is None or data['partition'] == partnum: + ptdata.append((path_to_kname(part_sysfs), data['partition'], + data['start'] * unit, data['size'] * unit,)) return ptdata +def get_part_table_type(device): + """ + check the type of partition table present on the specified device + returns None if no ptable was present or device could not be read + """ + # it is neccessary to look for the gpt signature first, then the dos + # signature, because a gpt formatted disk usually has a valid mbr to + # protect the disk from being modified by older partitioning tools + return ('gpt' if check_efi_signature(device) else + 'dos' if check_dos_signature(device) else None) + + +def check_dos_signature(device): + """ + check if there is a dos partition table signature present on device + """ + # the last 2 bytes of a dos partition table have the signature with the + # value 0xAA55. the dos partition table is always 0x200 bytes long, even if + # the underlying disk uses a larger logical block size, so the start of + # this signature must be at 0x1fe + # https://en.wikipedia.org/wiki/Master_boot_record#Sector_layout + return (is_block_device(device) and util.file_size(device) >= 0x200 and + (util.load_file(device, mode='rb', read_len=2, offset=0x1fe) == + b'\x55\xAA')) + + +def check_efi_signature(device): + """ + check if there is a gpt partition table signature present on device + """ + # the gpt partition table header is always on lba 1, regardless of the + # logical block size used by the underlying disk. therefore, a static + # offset cannot be used, the offset to the start of the table header is + # always the sector size of the disk + # the start of the gpt partition table header shoult have the signaure + # 'EFI PART'. + # https://en.wikipedia.org/wiki/GUID_Partition_Table + sector_size = get_blockdev_sector_size(device)[0] + return (is_block_device(device) and + util.file_size(device) >= 2 * sector_size and + (util.load_file(device, mode='rb', read_len=8, + offset=sector_size) == b'EFI PART')) + + +def is_extended_partition(device): + """ + check if the specified device path is a dos extended partition + """ + # an extended partition must be on a dos disk, must be a partition, must be + # within the first 4 partitions and will have a valid dos signature, + # because the format of the extended partition matches that of a real mbr + (parent_dev, part_number) = get_blockdev_for_partition(device) + return (get_part_table_type(parent_dev) in ['dos', 'msdos'] and + part_number is not None and int(part_number) <= 4 and + check_dos_signature(device)) + + def wipe_file(path, reader=None, buflen=4 * 1024 * 1024): - # wipe the existing file at path. - # if reader is provided, it will be called as a 'reader(buflen)' - # to provide data for each write. Otherwise, zeros are used. - # writes will be done in size of buflen. + """ + wipe the existing file at path. + if reader is provided, it will be called as a 'reader(buflen)' + to provide data for each write. Otherwise, zeros are used. + writes will be done in size of buflen. + """ if reader: readfunc = reader else: @@ -551,13 +738,11 @@ def readfunc(size): return buf + size = util.file_size(path) + LOG.debug("%s is %s bytes. wiping with buflen=%s", + path, size, buflen) + with open(path, "rb+") as fp: - # get the size by seeking to end. - fp.seek(0, 2) - size = fp.tell() - LOG.debug("%s is %s bytes. wiping with buflen=%s", - path, size, buflen) - fp.seek(0) while True: pbuf = readfunc(buflen) pos = fp.tell() @@ -574,16 +759,18 @@ def quick_zero(path, partitions=True): - # zero 1M at front, 1M at end, and 1M at front - # if this is a block device and partitions is true, then - # zero 1M at front and end of each partition. + """ + zero 1M at front, 1M at end, and 1M at front + if this is a block device and partitions is true, then + zero 1M at front and end of each partition. + """ buflen = 1024 count = 1024 zero_size = buflen * count offsets = [0, -zero_size] is_block = is_block_device(path) if not (is_block or os.path.isfile(path)): - raise ValueError("%s: not an existing file or block device") + raise ValueError("%s: not an existing file or block device", path) if partitions and is_block: ptdata = sysfs_partition_data(path) @@ -596,6 +783,9 @@ def zero_file_at_offsets(path, offsets, buflen=1024, count=1024, strict=False): + """ + write zeros to file at specified offsets + """ bmsg = "{path} (size={size}): " m_short = bmsg + "{tot} bytes from {offset} > size." m_badoff = bmsg + "invalid offset {offset}." @@ -657,15 +847,13 @@ if mode == "pvremove": # We need to use --force --force in case it's already in a volgroup and # pvremove doesn't want to remove it - cmds = [] - cmds.append(["pvremove", "--force", "--force", "--yes", path]) - cmds.append(["pvscan", "--cache"]) - cmds.append(["vgscan", "--mknodes", "--cache"]) + # If pvremove is run and there is no label on the system, # then it exits with 5. That is also okay, because we might be # wiping something that is already blank - for cmd in cmds: - util.subp(cmd, rcs=[0, 5], capture=True) + util.subp(['pvremove', '--force', '--force', '--yes', path], + rcs=[0, 5], capture=True) + lvm.lvm_scan() elif mode == "zero": wipe_file(path) elif mode == "random": === added file 'curtin/block/clear_holders.py' --- curtin/block/clear_holders.py 1970-01-01 00:00:00 +0000 +++ curtin/block/clear_holders.py 2016-10-03 18:55:20 +0000 @@ -0,0 +1,387 @@ +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Wesley Wiedenmeier +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +""" +This module provides a mechanism for shutting down virtual storage layers on +top of a block device, making it possible to reuse the block device without +having to reboot the system +""" + +import os + +from curtin import (block, udev, util) +from curtin.block import lvm +from curtin.log import LOG + + +def _define_handlers_registry(): + """ + returns instantiated dev_types + """ + return { + 'partition': {'shutdown': wipe_superblock, + 'ident': identify_partition}, + 'lvm': {'shutdown': shutdown_lvm, 'ident': identify_lvm}, + 'crypt': {'shutdown': shutdown_crypt, 'ident': identify_crypt}, + 'raid': {'shutdown': shutdown_mdadm, 'ident': identify_mdadm}, + 'bcache': {'shutdown': shutdown_bcache, 'ident': identify_bcache}, + 'disk': {'ident': lambda x: False, 'shutdown': wipe_superblock}, + } + + +def get_dmsetup_uuid(device): + """ + get the dm uuid for a specified dmsetup device + """ + blockdev = block.sysfs_to_devpath(device) + (out, _) = util.subp(['dmsetup', 'info', blockdev, '-C', '-o', 'uuid', + '--noheadings'], capture=True) + return out.strip() + + +def get_bcache_using_dev(device): + """ + Get the /sys/fs/bcache/ path of the bcache volume using specified device + """ + # FIXME: when block.bcache is written this should be moved there + sysfs_path = block.sys_block_path(device) + return os.path.realpath(os.path.join(sysfs_path, 'bcache', 'cache')) + + +def shutdown_bcache(device): + """ + Shut down bcache for specified bcache device + """ + bcache_shutdown_message = ('shutdown_bcache running on {} has determined ' + 'that the device has already been shut down ' + 'during handling of another bcache dev. ' + 'skipping'.format(device)) + if not os.path.exists(device): + LOG.info(bcache_shutdown_message) + return + + bcache_sysfs = get_bcache_using_dev(device) + if not os.path.exists(bcache_sysfs): + LOG.info(bcache_shutdown_message) + return + + LOG.debug('stopping bcache at: %s', bcache_sysfs) + util.write_file(os.path.join(bcache_sysfs, 'stop'), '1', mode=None) + + +def shutdown_lvm(device): + """ + Shutdown specified lvm device. + """ + device = block.sys_block_path(device) + # lvm devices have a dm directory that containes a file 'name' containing + # '{volume group}-{logical volume}'. The volume can be freed using lvremove + name_file = os.path.join(device, 'dm', 'name') + (vg_name, lv_name) = lvm.split_lvm_name(util.load_file(name_file)) + # use two --force flags here in case the volume group that this lv is + # attached two has been damaged + LOG.debug('running lvremove on %s/%s', vg_name, lv_name) + util.subp(['lvremove', '--force', '--force', + '{}/{}'.format(vg_name, lv_name)], rcs=[0, 5]) + # if that was the last lvol in the volgroup, get rid of volgroup + if len(lvm.get_lvols_in_volgroup(vg_name)) == 0: + util.subp(['vgremove', '--force', '--force', vg_name], rcs=[0, 5]) + # refresh lvmetad + lvm.lvm_scan() + + +def shutdown_crypt(device): + """ + Shutdown specified cryptsetup device + """ + blockdev = block.sysfs_to_devpath(device) + util.subp(['cryptsetup', 'remove', blockdev], capture=True) + + +def shutdown_mdadm(device): + """ + Shutdown specified mdadm device. + """ + blockdev = block.sysfs_to_devpath(device) + LOG.debug('using mdadm.mdadm_stop on dev: %s', blockdev) + block.mdadm.mdadm_stop(blockdev) + block.mdadm.mdadm_remove(blockdev) + + +def wipe_superblock(device): + """ + Wrapper for block.wipe_volume compatible with shutdown function interface + """ + blockdev = block.sysfs_to_devpath(device) + # when operating on a disk that used to have a dos part table with an + # extended partition, attempting to wipe the extended partition will fail + if block.is_extended_partition(blockdev): + LOG.info("extended partitions do not need wiping, so skipping: '%s'", + blockdev) + else: + LOG.info('wiping superblock on %s', blockdev) + block.wipe_volume(blockdev, mode='superblock') + + +def identify_lvm(device): + """ + determine if specified device is a lvm device + """ + return (block.path_to_kname(device).startswith('dm') and + get_dmsetup_uuid(device).startswith('LVM')) + + +def identify_crypt(device): + """ + determine if specified device is dm-crypt device + """ + return (block.path_to_kname(device).startswith('dm') and + get_dmsetup_uuid(device).startswith('CRYPT')) + + +def identify_mdadm(device): + """ + determine if specified device is a mdadm device + """ + return block.path_to_kname(device).startswith('md') + + +def identify_bcache(device): + """ + determine if specified device is a bcache device + """ + return block.path_to_kname(device).startswith('bcache') + + +def identify_partition(device): + """ + determine if specified device is a partition + """ + path = os.path.join(block.sys_block_path(device), 'partition') + return os.path.exists(path) + + +def get_holders(device): + """ + Look up any block device holders, return list of knames + """ + # block.sys_block_path works when given a /sys or /dev path + sysfs_path = block.sys_block_path(device) + # get holders + holders = os.listdir(os.path.join(sysfs_path, 'holders')) + LOG.debug("devname '%s' had holders: %s", device, holders) + return holders + + +def gen_holders_tree(device): + """ + generate a tree representing the current storage hirearchy above 'device' + """ + device = block.sys_block_path(device) + dev_name = block.path_to_kname(device) + # the holders for a device should consist of the devices in the holders/ + # dir in sysfs and any partitions on the device. this ensures that a + # storage tree starting from a disk will include all devices holding the + # disk's partitions + holder_paths = ([block.sys_block_path(h) for h in get_holders(device)] + + block.get_sysfs_partitions(device)) + # the DEV_TYPE registry contains a function under the key 'ident' for each + # device type entry that returns true if the device passed to it is of the + # correct type. there should never be a situation in which multiple + # identify functions return true. therefore, it will always work to take + # the device type with the first identify function that returns true as the + # device type for the current device. in the event that no identify + # functions return true, the device will be treated as a disk + # (DEFAULT_DEV_TYPE). the identify function for disk never returns true. + # the next() builtin in python will not raise a StopIteration exception if + # there is a default value defined + dev_type = next((k for k, v in DEV_TYPES.items() if v['ident'](device)), + DEFAULT_DEV_TYPE) + return { + 'device': device, 'dev_type': dev_type, 'name': dev_name, + 'holders': [gen_holders_tree(h) for h in holder_paths], + } + + +def plan_shutdown_holder_trees(holders_trees): + """ + plan best order to shut down holders in, taking into account high level + storage layers that may have many devices below them + + returns a sorted list of descriptions of storage config entries including + their path in /sys/block and their dev type + + can accept either a single storage tree or a list of storage trees assumed + to start at an equal place in storage hirearchy (i.e. a list of trees + starting from disk) + """ + # holds a temporary registry of holders to allow cross references + # key = device sysfs path, value = {} of priority level, shutdown function + reg = {} + + # normalize to list of trees + if not isinstance(holders_trees, (list, tuple)): + holders_trees = [holders_trees] + + def flatten_holders_tree(tree, level=0): + """ + add entries from holders tree to registry with level key corresponding + to how many layers from raw disks the current device is at + """ + device = tree['device'] + + # always go with highest level if current device has been + # encountered already. since the device and everything above it is + # re-added to the registry it ensures that any increase of level + # required here will propagate down the tree + # this handles a scenario like mdadm + bcache, where the backing + # device for bcache is a 3nd level item like mdadm, but the cache + # device is 1st level (disk) or second level (partition), ensuring + # that the bcache item is always considered higher level than + # anything else regardless of whether it was added to the tree via + # the cache device or backing device first + if device in reg: + level = max(reg[device]['level'], level) + + reg[device] = {'level': level, 'device': device, + 'dev_type': tree['dev_type']} + + # handle holders above this level + for holder in tree['holders']: + flatten_holders_tree(holder, level=level + 1) + + # flatten the holders tree into the registry + for holders_tree in holders_trees: + flatten_holders_tree(holders_tree) + + # return list of entry dicts with highest level first + return [reg[k] for k in sorted(reg, key=lambda x: reg[x]['level'] * -1)] + + +def format_holders_tree(holders_tree): + """ + draw a nice dirgram of the holders tree + """ + # spacer styles based on output of 'tree --charset=ascii' + spacers = (('`-- ', ' ' * 4), ('|-- ', '|' + ' ' * 3)) + + def format_tree(tree): + """ + format entry and any subentries + """ + result = [tree['name']] + holders = tree['holders'] + for (holder_no, holder) in enumerate(holders): + spacer_style = spacers[min(len(holders) - (holder_no + 1), 1)] + subtree_lines = format_tree(holder) + for (line_no, line) in enumerate(subtree_lines): + result.append(spacer_style[min(line_no, 1)] + line) + return result + + return '\n'.join(format_tree(holders_tree)) + + +def get_holder_types(tree): + """ + get flattened list of types of holders in holders tree and the devices + they correspond to + """ + types = {(tree['dev_type'], tree['device'])} + for holder in tree['holders']: + types.update(get_holder_types(holder)) + return types + + +def assert_clear(base_paths): + """ + Check if all paths in base_paths are clear to use + """ + valid = ('disk', 'partition') + if not isinstance(base_paths, (list, tuple)): + base_paths = [base_paths] + base_paths = [block.sys_block_path(path) for path in base_paths] + for holders_tree in [gen_holders_tree(p) for p in base_paths]: + if any(holder_type not in valid and path not in base_paths + for (holder_type, path) in get_holder_types(holders_tree)): + raise OSError('Storage not clear, remaining:\n{}' + .format(format_holders_tree(holders_tree))) + + +def clear_holders(base_paths, try_preserve=False): + """ + Clear all storage layers depending on the devices specified in 'base_paths' + A single device or list of devices can be specified. + Device paths can be specified either as paths in /dev or /sys/block + Will throw OSError if any holders could not be shut down + """ + # handle single path + if not isinstance(base_paths, (list, tuple)): + base_paths = [base_paths] + + # get current holders and plan how to shut them down + holder_trees = [gen_holders_tree(path) for path in base_paths] + LOG.info('Current device storage tree:\n%s', + '\n'.join(format_holders_tree(tree) for tree in holder_trees)) + ordered_devs = plan_shutdown_holder_trees(holder_trees) + + # run shutdown functions + for dev_info in ordered_devs: + dev_type = DEV_TYPES.get(dev_info['dev_type']) + shutdown_function = dev_type.get('shutdown') + if not shutdown_function: + continue + if try_preserve and shutdown_function in DATA_DESTROYING_HANDLERS: + LOG.info('shutdown function for holder type: %s is destructive. ' + 'attempting to preserve data, so not skipping' % + dev_info['dev_type']) + continue + LOG.info("shutdown running on holder type: '%s' syspath: '%s'", + dev_info['dev_type'], dev_info['device']) + shutdown_function(dev_info['device']) + udev.udevadm_settle() + + +def start_clear_holders_deps(): + """ + prepare system for clear holders to be able to scan old devices + """ + # a mdadm scan has to be started in case there is a md device that needs to + # be detected. if the scan fails, it is either because there are no mdadm + # devices on the system, or because there is a mdadm device in a damaged + # state that could not be started. due to the nature of mdadm tools, it is + # difficult to know which is the case. if any errors did occur, then ignore + # them, since no action needs to be taken if there were no mdadm devices on + # the system, and in the case where there is some mdadm metadata on a disk, + # but there was not enough to start the array, the call to wipe_volume on + # all disks and partitions should be sufficient to remove the mdadm + # metadata + block.mdadm.mdadm_assemble(scan=True, ignore_errors=True) + # the bcache module needs to be present to properly detect bcache devs + # on some systems (precise without hwe kernel) it may not be possible to + # lad the bcache module bcause it is not present in the kernel. if this + # happens then there is no need to halt installation, as the bcache devices + # will never appear and will never prevent the disk from being reformatted + util.subp(['modprobe', 'bcache'], rcs=[0, 1]) + + +# anything that is not identified can assumed to be a 'disk' or similar +DEFAULT_DEV_TYPE = 'disk' +# handlers that should not be run if an attempt is being made to preserve data +DATA_DESTROYING_HANDLERS = [wipe_superblock] +# types of devices that could be encountered by clear holders and functions to +# identify them and shut them down +DEV_TYPES = _define_handlers_registry() === added file 'curtin/block/lvm.py' --- curtin/block/lvm.py 1970-01-01 00:00:00 +0000 +++ curtin/block/lvm.py 2016-10-03 18:55:20 +0000 @@ -0,0 +1,96 @@ +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Wesley Wiedenmeier +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +""" +This module provides some helper functions for manipulating lvm devices +""" + +from curtin import util +from curtin.log import LOG +import os + +# separator to use for lvm/dm tools +_SEP = '=' + + +def _filter_lvm_info(lvtool, match_field, query_field, match_key): + """ + filter output of pv/vg/lvdisplay tools + """ + (out, _) = util.subp([lvtool, '-C', '--separator', _SEP, '--noheadings', + '-o', ','.join([match_field, query_field])], + capture=True) + return [qf for (mf, qf) in + [l.strip().split(_SEP) for l in out.strip().splitlines()] + if mf == match_key] + + +def get_pvols_in_volgroup(vg_name): + """ + get physical volumes used by volgroup + """ + return _filter_lvm_info('pvdisplay', 'vg_name', 'pv_name', vg_name) + + +def get_lvols_in_volgroup(vg_name): + """ + get logical volumes in volgroup + """ + return _filter_lvm_info('lvdisplay', 'vg_name', 'lv_name', vg_name) + + +def split_lvm_name(full): + """ + split full lvm name into tuple of (volgroup, lv_name) + """ + # 'dmsetup splitname' is the authoratative source for lvm name parsing + (out, _) = util.subp(['dmsetup', 'splitname', full, '-c', '--noheadings', + '--separator', _SEP, '-o', 'vg_name,lv_name'], + capture=True) + return out.strip().split(_SEP) + + +def lvmetad_running(): + """ + check if lvmetad is running + """ + return os.path.exists(os.environ.get('LVM_LVMETAD_PIDFILE', + '/run/lvmetad.pid')) + + +def lvm_scan(): + """ + run full scan for volgroups, logical volumes and physical volumes + """ + # the lvm tools lvscan, vgscan and pvscan on ubuntu precise do not + # support the flag --cache. the flag is present for the tools in ubuntu + # trusty and later. since lvmetad is used in current releases of + # ubuntu, the --cache flag is needed to ensure that the data cached by + # lvmetad is updated. + + # before appending the cache flag though, check if lvmetad is running. this + # ensures that we do the right thing even if lvmetad is supported but is + # not running + release = util.lsb_release().get('codename') + if release in [None, 'UNAVAILABLE']: + LOG.warning('unable to find release number, assuming xenial or later') + release = 'xenial' + + for cmd in [['pvscan'], ['vgscan', '--mknodes']]: + if release != 'precise' and lvmetad_running(): + cmd.append('--cache') + util.subp(cmd, capture=True) === modified file 'curtin/block/mdadm.py' --- curtin/block/mdadm.py 2016-05-10 16:13:29 +0000 +++ curtin/block/mdadm.py 2016-10-03 18:55:20 +0000 @@ -28,7 +28,7 @@ from subprocess import CalledProcessError from curtin.block import (dev_short, dev_path, is_valid_device, sys_block_path) -from curtin import util +from curtin import (util, udev) from curtin.log import LOG NOSPARE_RAID_LEVELS = [ @@ -117,21 +117,34 @@ # -def mdadm_assemble(md_devname=None, devices=[], spares=[], scan=False): +def mdadm_assemble(md_devname=None, devices=[], spares=[], scan=False, + ignore_errors=False): # md_devname is a /dev/XXXX # devices is non-empty list of /dev/xxx # if spares is non-empt list append of /dev/xxx cmd = ["mdadm", "--assemble"] if scan: - cmd += ['--scan'] + cmd += ['--scan', '-v'] else: valid_mdname(md_devname) cmd += [md_devname, "--run"] + devices if spares: cmd += spares - util.subp(cmd, capture=True, rcs=[0, 1, 2]) - util.subp(["udevadm", "settle"]) + try: + # mdadm assemble returns 1 when no arrays are found. this might not be + # an error depending on the situation this function was called in, so + # accept a return code of 1 + # mdadm assemble returns 2 when called on an array that is already + # assembled. this is not an error, so accept return code of 2 + # all other return codes can be accepted with ignore_error set to true + util.subp(cmd, capture=True, rcs=[0, 1, 2]) + except util.ProcessExecutionError: + LOG.warning("mdadm_assemble had unexpected return code") + if not ignore_errors: + raise + + udev.udevadm_settle() def mdadm_create(md_devname, raidlevel, devices, spares=None, md_name=""): === modified file 'curtin/block/mkfs.py' --- curtin/block/mkfs.py 2016-05-10 16:13:29 +0000 +++ curtin/block/mkfs.py 2016-10-03 18:55:20 +0000 @@ -78,6 +78,7 @@ "swap": "--uuid"}, "force": {"btrfs": "--force", "ext": "-F", + "fat": "-I", "ntfs": "--force", "reiserfs": "-f", "swap": "--force", @@ -91,6 +92,7 @@ "btrfs": "--sectorsize", "ext": "-b", "fat": "-S", + "xfs": "-s", "ntfs": "--sector-size", "reiserfs": "--block-size"} } @@ -165,12 +167,15 @@ # use device logical block size to ensure properly formated filesystems (logical_bsize, physical_bsize) = block.get_blockdev_sector_size(path) if logical_bsize > 512: + lbs_str = ('size={}'.format(logical_bsize) if fs_family == "xfs" + else str(logical_bsize)) cmd.extend(get_flag_mapping("sectorsize", fs_family, - param=str(logical_bsize), - strict=strict)) - # mkfs.vfat doesn't calculate this right for non-512b sector size - # lp:1569576 , d-i uses the same setting. - cmd.extend(["-s", "1"]) + param=lbs_str, strict=strict)) + + if fs_family == 'fat': + # mkfs.vfat doesn't calculate this right for non-512b sector size + # lp:1569576 , d-i uses the same setting. + cmd.extend(["-s", "1"]) if force: cmd.extend(get_flag_mapping("force", fs_family, strict=strict)) === modified file 'curtin/commands/apply_net.py' --- curtin/commands/apply_net.py 2016-05-10 16:13:29 +0000 +++ curtin/commands/apply_net.py 2016-10-03 18:55:20 +0000 @@ -26,6 +26,57 @@ LOG = log.LOG +IFUPDOWN_IPV6_MTU_PRE_HOOK = """#!/bin/bash -e +# injected by curtin installer + +[ "${IFACE}" != "lo" ] || exit 0 + +# Trigger only if MTU configured +[ -n "${IF_MTU}" ] || exit 0 + +read CUR_DEV_MTU /run/network/${IFACE}_dev.mtu +[ -n "${CUR_IPV6_MTU}" ] && + echo ${CUR_IPV6_MTU} > /run/network/${IFACE}_ipv6.mtu +exit 0 +""" + +IFUPDOWN_IPV6_MTU_POST_HOOK = """#!/bin/bash -e +# injected by curtin installer + +[ "${IFACE}" != "lo" ] || exit 0 + +# Trigger only if MTU configured +[ -n "${IF_MTU}" ] || exit 0 + +read PRE_DEV_MTU /proc/sys/net/ipv6/conf/${IFACE}/mtu ||: + +elif [ "${ADDRFAM}" = "inet" ]; then + # handle the clobber case where inet mtu changes v6 mtu. + # ifupdown will already have set dev mtu, so lower mtu + # if needed. If v6 mtu was larger, it get's clamped down + # to the dev MTU value. + if [ ${PRE_IPV6_MTU} -lt ${CUR_IPV6_MTU} ]; then + # sysctl -q -e -w net.ipv6.conf.${IFACE}.mtu=${PRE_IPV6_MTU} + echo ${PRE_IPV6_MTU} >/proc/sys/net/ipv6/conf/${IFACE}/mtu ||: + fi +fi +exit 0 +""" + def apply_net(target, network_state=None, network_config=None): if network_state is None and network_config is None: @@ -45,6 +96,108 @@ net.render_network_state(target=target, network_state=ns) + _maybe_remove_legacy_eth0(target) + LOG.info('Attempting to remove ipv6 privacy extensions') + _disable_ipv6_privacy_extensions(target) + _patch_ifupdown_ipv6_mtu_hook(target) + + +def _patch_ifupdown_ipv6_mtu_hook(target, + prehookfn="etc/network/if-pre-up.d/mtuipv6", + posthookfn="etc/network/if-up.d/mtuipv6"): + + contents = { + 'prehook': IFUPDOWN_IPV6_MTU_PRE_HOOK, + 'posthook': IFUPDOWN_IPV6_MTU_POST_HOOK, + } + + hookfn = { + 'prehook': prehookfn, + 'posthook': posthookfn, + } + + for hook in ['prehook', 'posthook']: + fn = hookfn[hook] + cfg = util.target_path(target, path=fn) + LOG.info('Injecting fix for ipv6 mtu settings: %s', cfg) + util.write_file(cfg, contents[hook], mode=0o755) + + +def _disable_ipv6_privacy_extensions(target, + path="etc/sysctl.d/10-ipv6-privacy.conf"): + + """Ubuntu server image sets a preference to use IPv6 privacy extensions + by default; this races with the cloud-image desire to disable them. + Resolve this by allowing the cloud-image setting to win. """ + + cfg = util.target_path(target, path=path) + if not os.path.exists(cfg): + LOG.warn('Failed to find ipv6 privacy conf file %s', cfg) + return + + bmsg = "Disabling IPv6 privacy extensions config may not apply." + try: + contents = util.load_file(cfg) + known_contents = ["net.ipv6.conf.all.use_tempaddr = 2", + "net.ipv6.conf.default.use_tempaddr = 2"] + lines = [f.strip() for f in contents.splitlines() + if not f.startswith("#")] + if lines == known_contents: + LOG.info('deleting file: %s', cfg) + util.del_file(cfg) + msg = "removed %s with known contents" % cfg + curtin_contents = '\n'.join( + ["# IPv6 Privacy Extensions (RFC 4941)", + "# Disabled by curtin", + "# net.ipv6.conf.all.use_tempaddr = 2", + "# net.ipv6.conf.default.use_tempaddr = 2"]) + util.write_file(cfg, curtin_contents) + else: + LOG.info('skipping, content didnt match') + LOG.debug("found content:\n%s", lines) + LOG.debug("expected contents:\n%s", known_contents) + msg = (bmsg + " '%s' exists with user configured content." % cfg) + except: + msg = bmsg + " %s exists, but could not be read." % cfg + LOG.exception(msg) + return + + +def _maybe_remove_legacy_eth0(target, + path="etc/network/interfaces.d/eth0.cfg"): + """Ubuntu cloud images previously included a 'eth0.cfg' that had + hard coded content. That file would interfere with the rendered + configuration if it was present. + + if the file does not exist do nothing. + If the file exists: + - with known content, remove it and warn + - with unknown content, leave it and warn + """ + + cfg = util.target_path(target, path=path) + if not os.path.exists(cfg): + LOG.warn('Failed to find legacy network conf file %s', cfg) + return + + bmsg = "Dynamic networking config may not apply." + try: + contents = util.load_file(cfg) + known_contents = ["auto eth0", "iface eth0 inet dhcp"] + lines = [f.strip() for f in contents.splitlines() + if not f.startswith("#")] + if lines == known_contents: + util.del_file(cfg) + msg = "removed %s with known contents" % cfg + else: + msg = (bmsg + " '%s' exists with user configured content." % cfg) + except: + msg = bmsg + " %s exists, but could not be read." % cfg + LOG.exception(msg) + return + + LOG.warn(msg) + def apply_net_main(args): # curtin apply_net [--net-state=/config/netstate.yml] [--target=/] @@ -76,8 +229,10 @@ apply_net(target=state['target'], network_state=state['network_state'], network_config=state['network_config']) + except Exception: LOG.exception('failed to apply network config') + return 1 LOG.info('Applied network configuration successfully') sys.exit(0) @@ -90,7 +245,7 @@ 'metavar': 'NETSTATE', 'action': 'store', 'default': os.environ.get('OUTPUT_NETWORK_STATE')}), (('-t', '--target'), - {'help': ('target filesystem root to add swap file to. ' + {'help': ('target filesystem root to configure networking to. ' 'default is env["TARGET_MOUNT_POINT"]'), 'metavar': 'TARGET', 'action': 'store', 'default': os.environ.get('TARGET_MOUNT_POINT')}), === added file 'curtin/commands/apt_config.py' --- curtin/commands/apt_config.py 1970-01-01 00:00:00 +0000 +++ curtin/commands/apt_config.py 2016-10-03 18:55:20 +0000 @@ -0,0 +1,668 @@ +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Christian Ehrhardt +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . +""" +apt.py +Handle the setup of apt related tasks like proxies, mirrors, repositories. +""" + +import argparse +import glob +import os +import re +import sys +import yaml + +from curtin.log import LOG +from curtin import (config, util, gpg) + +from . import populate_one_subcmd + +# this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar') +ADD_APT_REPO_MATCH = r"^[\w-]+:\w" + +# place where apt stores cached repository data +APT_LISTS = "/var/lib/apt/lists" + +# Files to store proxy information +APT_CONFIG_FN = "/etc/apt/apt.conf.d/94curtin-config" +APT_PROXY_FN = "/etc/apt/apt.conf.d/90curtin-aptproxy" + +# Default keyserver to use +DEFAULT_KEYSERVER = "keyserver.ubuntu.com" + +# Default archive mirrors +PRIMARY_ARCH_MIRRORS = {"PRIMARY": "http://archive.ubuntu.com/ubuntu/", + "SECURITY": "http://security.ubuntu.com/ubuntu/"} +PORTS_MIRRORS = {"PRIMARY": "http://ports.ubuntu.com/ubuntu-ports", + "SECURITY": "http://ports.ubuntu.com/ubuntu-ports"} +PRIMARY_ARCHES = ['amd64', 'i386'] +PORTS_ARCHES = ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el'] + + +def get_default_mirrors(arch=None): + """returns the default mirrors for the target. These depend on the + architecture, for more see: + https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports""" + if arch is None: + arch = util.get_architecture() + if arch in PRIMARY_ARCHES: + return PRIMARY_ARCH_MIRRORS.copy() + if arch in PORTS_ARCHES: + return PORTS_MIRRORS.copy() + raise ValueError("No default mirror known for arch %s" % arch) + + +def handle_apt(cfg, target=None): + """ handle_apt + process the config for apt_config. This can be called from + curthooks if a global apt config was provided or via the "apt" + standalone command. + """ + release = util.lsb_release(target=target)['codename'] + arch = util.get_architecture(target) + mirrors = find_apt_mirror_info(cfg, arch) + LOG.debug("Apt Mirror info: %s", mirrors) + + apply_debconf_selections(cfg, target) + + if not config.value_as_boolean(cfg.get('preserve_sources_list', + True)): + generate_sources_list(cfg, release, mirrors, target) + rename_apt_lists(mirrors, target) + + try: + apply_apt_proxy_config(cfg, target + APT_PROXY_FN, + target + APT_CONFIG_FN) + except (IOError, OSError): + LOG.exception("Failed to apply proxy or apt config info:") + + # Process 'apt_source -> sources {dict}' + if 'sources' in cfg: + params = mirrors + params['RELEASE'] = release + params['MIRROR'] = mirrors["MIRROR"] + + matcher = None + matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH) + if matchcfg: + matcher = re.compile(matchcfg).search + + add_apt_sources(cfg['sources'], target, + template_params=params, aa_repo_match=matcher) + + +def debconf_set_selections(selections, target=None): + util.subp(['debconf-set-selections'], data=selections, target=target, + capture=True) + + +def dpkg_reconfigure(packages, target=None): + # For any packages that are already installed, but have preseed data + # we populate the debconf database, but the filesystem configuration + # would be preferred on a subsequent dpkg-reconfigure. + # so, what we have to do is "know" information about certain packages + # to unconfigure them. + unhandled = [] + to_config = [] + for pkg in packages: + if pkg in CONFIG_CLEANERS: + LOG.debug("unconfiguring %s", pkg) + CONFIG_CLEANERS[pkg](target) + to_config.append(pkg) + else: + unhandled.append(pkg) + + if len(unhandled): + LOG.warn("The following packages were installed and preseeded, " + "but cannot be unconfigured: %s", unhandled) + + if len(to_config): + util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] + + list(to_config), data=None, target=target, capture=True) + + +def apply_debconf_selections(cfg, target=None): + """apply_debconf_selections - push content to debconf""" + # debconf_selections: + # set1: | + # cloud-init cloud-init/datasources multiselect MAAS + # set2: pkg pkg/value string bar + selsets = cfg.get('debconf_selections') + if not selsets: + LOG.debug("debconf_selections was not set in config") + return + + selections = '\n'.join( + [selsets[key] for key in sorted(selsets.keys())]) + debconf_set_selections(selections.encode() + b"\n", target=target) + + # get a complete list of packages listed in input + pkgs_cfgd = set() + for key, content in selsets.items(): + for line in content.splitlines(): + if line.startswith("#"): + continue + pkg = re.sub(r"[:\s].*", "", line) + pkgs_cfgd.add(pkg) + + pkgs_installed = util.get_installed_packages(target) + + LOG.debug("pkgs_cfgd: %s", pkgs_cfgd) + LOG.debug("pkgs_installed: %s", pkgs_installed) + need_reconfig = pkgs_cfgd.intersection(pkgs_installed) + + if len(need_reconfig) == 0: + LOG.debug("no need for reconfig") + return + + dpkg_reconfigure(need_reconfig, target=target) + + +def clean_cloud_init(target): + """clean out any local cloud-init config""" + flist = glob.glob( + util.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*")) + + LOG.debug("cleaning cloud-init config from: %s", flist) + for dpkg_cfg in flist: + os.unlink(dpkg_cfg) + + +def mirrorurl_to_apt_fileprefix(mirror): + """ mirrorurl_to_apt_fileprefix + Convert a mirror url to the file prefix used by apt on disk to + store cache information for that mirror. + To do so do: + - take off ???:// + - drop tailing / + - convert in string / to _ + """ + string = mirror + if string.endswith("/"): + string = string[0:-1] + pos = string.find("://") + if pos >= 0: + string = string[pos + 3:] + string = string.replace("/", "_") + return string + + +def rename_apt_lists(new_mirrors, target=None): + """rename_apt_lists - rename apt lists to preserve old cache data""" + default_mirrors = get_default_mirrors(util.get_architecture(target)) + + pre = util.target_path(target, APT_LISTS) + for (name, omirror) in default_mirrors.items(): + nmirror = new_mirrors.get(name) + if not nmirror: + continue + + oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror) + nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror) + if oprefix == nprefix: + continue + olen = len(oprefix) + for filename in glob.glob("%s_*" % oprefix): + newname = "%s%s" % (nprefix, filename[olen:]) + LOG.debug("Renaming apt list %s to %s", filename, newname) + try: + os.rename(filename, newname) + except OSError: + # since this is a best effort task, warn with but don't fail + LOG.warn("Failed to rename apt list:", exc_info=True) + + +def mirror_to_placeholder(tmpl, mirror, placeholder): + """ mirror_to_placeholder + replace the specified mirror in a template with a placeholder string + Checks for existance of the expected mirror and warns if not found + """ + if mirror not in tmpl: + LOG.warn("Expected mirror '%s' not found in: %s", mirror, tmpl) + return tmpl.replace(mirror, placeholder) + + +def map_known_suites(suite): + """there are a few default names which will be auto-extended. + This comes at the inability to use those names literally as suites, + but on the other hand increases readability of the cfg quite a lot""" + mapping = {'updates': '$RELEASE-updates', + 'backports': '$RELEASE-backports', + 'security': '$RELEASE-security', + 'proposed': '$RELEASE-proposed', + 'release': '$RELEASE'} + try: + retsuite = mapping[suite] + except KeyError: + retsuite = suite + return retsuite + + +def disable_suites(disabled, src, release): + """reads the config for suites to be disabled and removes those + from the template""" + if not disabled: + return src + + retsrc = src + for suite in disabled: + suite = map_known_suites(suite) + releasesuite = util.render_string(suite, {'RELEASE': release}) + LOG.debug("Disabling suite %s as %s", suite, releasesuite) + + newsrc = "" + for line in retsrc.splitlines(True): + if line.startswith("#"): + newsrc += line + continue + + # sources.list allow options in cols[1] which can have spaces + # so the actual suite can be [2] or later. example: + # deb [ arch=amd64,armel k=v ] http://example.com/debian + cols = line.split() + if len(cols) > 1: + pcol = 2 + if cols[1].startswith("["): + for col in cols[1:]: + pcol += 1 + if col.endswith("]"): + break + + if cols[pcol] == releasesuite: + line = '# suite disabled by curtin: %s' % line + newsrc += line + retsrc = newsrc + + return retsrc + + +def generate_sources_list(cfg, release, mirrors, target=None): + """ generate_sources_list + create a source.list file based on a custom or default template + by replacing mirrors and release in the template + """ + default_mirrors = get_default_mirrors(util.get_architecture(target)) + aptsrc = "/etc/apt/sources.list" + params = {'RELEASE': release} + for k in mirrors: + params[k] = mirrors[k] + + tmpl = cfg.get('sources_list', None) + if tmpl is None: + LOG.info("No custom template provided, fall back to modify" + "mirrors in %s on the target system", aptsrc) + tmpl = util.load_file(util.target_path(target, aptsrc)) + # Strategy if no custom template was provided: + # - Only replacing mirrors + # - no reason to replace "release" as it is from target anyway + # - The less we depend upon, the more stable this is against changes + # - warn if expected original content wasn't found + tmpl = mirror_to_placeholder(tmpl, default_mirrors['PRIMARY'], + "$MIRROR") + tmpl = mirror_to_placeholder(tmpl, default_mirrors['SECURITY'], + "$SECURITY") + + orig = util.target_path(target, aptsrc) + if os.path.exists(orig): + os.rename(orig, orig + ".curtin.old") + + rendered = util.render_string(tmpl, params) + disabled = disable_suites(cfg.get('disable_suites'), rendered, release) + util.write_file(util.target_path(target, aptsrc), disabled, mode=0o644) + + # protect the just generated sources.list from cloud-init + cloudfile = "/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg" + # this has to work with older cloud-init as well, so use old key + cloudconf = yaml.dump({'apt_preserve_sources_list': True}, indent=1) + try: + util.write_file(util.target_path(target, cloudfile), + cloudconf, mode=0o644) + except IOError: + LOG.exception("Failed to protect source.list from cloud-init in (%s)", + util.target_path(target, cloudfile)) + raise + + +def add_apt_key_raw(key, target=None): + """ + actual adding of a key as defined in key argument + to the system + """ + LOG.debug("Adding key:\n'%s'", key) + try: + util.subp(['apt-key', 'add', '-'], data=key.encode(), target=target) + except util.ProcessExecutionError: + LOG.exception("failed to add apt GPG Key to apt keyring") + raise + + +def add_apt_key(ent, target=None): + """ + Add key to the system as defined in ent (if any). + Supports raw keys or keyid's + The latter will as a first step fetched to get the raw key + """ + if 'keyid' in ent and 'key' not in ent: + keyserver = DEFAULT_KEYSERVER + if 'keyserver' in ent: + keyserver = ent['keyserver'] + + ent['key'] = gpg.getkeybyid(ent['keyid'], keyserver) + + if 'key' in ent: + add_apt_key_raw(ent['key'], target) + + +def add_apt_sources(srcdict, target=None, template_params=None, + aa_repo_match=None): + """ + add entries in /etc/apt/sources.list.d for each abbreviated + sources.list entry in 'srcdict'. When rendering template, also + include the values in dictionary searchList + """ + if template_params is None: + template_params = {} + + if aa_repo_match is None: + raise ValueError('did not get a valid repo matcher') + + if not isinstance(srcdict, dict): + raise TypeError('unknown apt format: %s' % (srcdict)) + + for filename in srcdict: + ent = srcdict[filename] + if 'filename' not in ent: + ent['filename'] = filename + + add_apt_key(ent, target) + + if 'source' not in ent: + continue + source = ent['source'] + source = util.render_string(source, template_params) + + if not ent['filename'].startswith("/"): + ent['filename'] = os.path.join("/etc/apt/sources.list.d/", + ent['filename']) + if not ent['filename'].endswith(".list"): + ent['filename'] += ".list" + + if aa_repo_match(source): + try: + with util.ChrootableTarget( + target, sys_resolvconf=True) as in_chroot: + in_chroot.subp(["add-apt-repository", source]) + except util.ProcessExecutionError: + LOG.exception("add-apt-repository failed.") + raise + continue + + sourcefn = util.target_path(target, ent['filename']) + try: + contents = "%s\n" % (source) + util.write_file(sourcefn, contents, omode="a") + except IOError as detail: + LOG.exception("failed write to file %s: %s", sourcefn, detail) + raise + + util.apt_update(target=target, force=True, + comment="apt-source changed config") + + return + + +def search_for_mirror(candidates): + """ + Search through a list of mirror urls for one that works + This needs to return quickly. + """ + if candidates is None: + return None + + LOG.debug("search for mirror in candidates: '%s'", candidates) + for cand in candidates: + try: + if util.is_resolvable_url(cand): + LOG.debug("found working mirror: '%s'", cand) + return cand + except Exception: + pass + return None + + +def update_mirror_info(pmirror, smirror, arch): + """sets security mirror to primary if not defined. + returns defaults if no mirrors are defined""" + if pmirror is not None: + if smirror is None: + smirror = pmirror + return {'PRIMARY': pmirror, + 'SECURITY': smirror} + return get_default_mirrors(arch) + + +def get_arch_mirrorconfig(cfg, mirrortype, arch): + """out of a list of potential mirror configurations select + and return the one matching the architecture (or default)""" + # select the mirror specification (if-any) + mirror_cfg_list = cfg.get(mirrortype, None) + if mirror_cfg_list is None: + return None + + # select the specification matching the target arch + default = None + for mirror_cfg_elem in mirror_cfg_list: + arches = mirror_cfg_elem.get("arches") + if arch in arches: + return mirror_cfg_elem + if "default" in arches: + default = mirror_cfg_elem + return default + + +def get_mirror(cfg, mirrortype, arch): + """pass the three potential stages of mirror specification + returns None is neither of them found anything otherwise the first + hit is returned""" + mcfg = get_arch_mirrorconfig(cfg, mirrortype, arch) + if mcfg is None: + return None + + # directly specified + mirror = mcfg.get("uri", None) + + # fallback to search if specified + if mirror is None: + # list of mirrors to try to resolve + mirror = search_for_mirror(mcfg.get("search", None)) + + return mirror + + +def find_apt_mirror_info(cfg, arch=None): + """find_apt_mirror_info + find an apt_mirror given the cfg provided. + It can check for separate config of primary and security mirrors + If only primary is given security is assumed to be equal to primary + If the generic apt_mirror is given that is defining for both + """ + + if arch is None: + arch = util.get_architecture() + LOG.debug("got arch for mirror selection: %s", arch) + pmirror = get_mirror(cfg, "primary", arch) + LOG.debug("got primary mirror: %s", pmirror) + smirror = get_mirror(cfg, "security", arch) + LOG.debug("got security mirror: %s", smirror) + + # Note: curtin has no cloud-datasource fallback + + mirror_info = update_mirror_info(pmirror, smirror, arch) + + # less complex replacements use only MIRROR, derive from primary + mirror_info["MIRROR"] = mirror_info["PRIMARY"] + + return mirror_info + + +def apply_apt_proxy_config(cfg, proxy_fname, config_fname): + """apply_apt_proxy_config + Applies any apt*proxy config from if specified + """ + # Set up any apt proxy + cfgs = (('proxy', 'Acquire::http::Proxy "%s";'), + ('http_proxy', 'Acquire::http::Proxy "%s";'), + ('ftp_proxy', 'Acquire::ftp::Proxy "%s";'), + ('https_proxy', 'Acquire::https::Proxy "%s";')) + + proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)] + if len(proxies): + LOG.debug("write apt proxy info to %s", proxy_fname) + util.write_file(proxy_fname, '\n'.join(proxies) + '\n') + elif os.path.isfile(proxy_fname): + util.del_file(proxy_fname) + LOG.debug("no apt proxy configured, removed %s", proxy_fname) + + if cfg.get('conf', None): + LOG.debug("write apt config info to %s", config_fname) + util.write_file(config_fname, cfg.get('conf')) + elif os.path.isfile(config_fname): + util.del_file(config_fname) + LOG.debug("no apt config configured, removed %s", config_fname) + + +def apt_command(args): + """ Main entry point for curtin apt-config standalone command + This does not read the global config as handled by curthooks, but + instead one can specify a different "target" and a new cfg via --config + """ + cfg = config.load_command_config(args, {}) + + if args.target is not None: + target = args.target + else: + state = util.load_command_environment() + target = state['target'] + + if target is None: + sys.stderr.write("Unable to find target. " + "Use --target or set TARGET_MOUNT_POINT\n") + sys.exit(2) + + apt_cfg = cfg.get("apt") + # if no apt config section is available, do nothing + if apt_cfg is not None: + LOG.debug("Handling apt to target %s with config %s", + target, apt_cfg) + try: + with util.ChrootableTarget(target, sys_resolvconf=True): + handle_apt(apt_cfg, target) + except (RuntimeError, TypeError, ValueError, IOError): + LOG.exception("Failed to configure apt features '%s'", apt_cfg) + sys.exit(1) + else: + LOG.info("No apt config provided, skipping") + + sys.exit(0) + + +def translate_old_apt_features(cfg): + """translate the few old apt related features into the new config format""" + predef_apt_cfg = cfg.get("apt") + if predef_apt_cfg is None: + cfg['apt'] = {} + predef_apt_cfg = cfg.get("apt") + + if cfg.get('apt_proxy') is not None: + if predef_apt_cfg.get('proxy') is not None: + msg = ("Error in apt_proxy configuration: " + "old and new format of apt features " + "are mutually exclusive") + LOG.error(msg) + raise ValueError(msg) + + cfg['apt']['proxy'] = cfg.get('apt_proxy') + LOG.debug("Transferred %s into new format: %s", cfg.get('apt_proxy'), + cfg.get('apte')) + del cfg['apt_proxy'] + + if cfg.get('apt_mirrors') is not None: + if predef_apt_cfg.get('mirrors') is not None: + msg = ("Error in apt_mirror configuration: " + "old and new format of apt features " + "are mutually exclusive") + LOG.error(msg) + raise ValueError(msg) + + old = cfg.get('apt_mirrors') + cfg['apt']['primary'] = [{"arches": ["default"], + "uri": old.get('ubuntu_archive')}] + cfg['apt']['security'] = [{"arches": ["default"], + "uri": old.get('ubuntu_security')}] + LOG.debug("Transferred %s into new format: %s", cfg.get('apt_mirror'), + cfg.get('apt')) + del cfg['apt_mirrors'] + # to work this also needs to disable the default protection + psl = predef_apt_cfg.get('preserve_sources_list') + if psl is not None: + if config.value_as_boolean(psl) is True: + msg = ("Error in apt_mirror configuration: " + "apt_mirrors and preserve_sources_list: True " + "are mutually exclusive") + LOG.error(msg) + raise ValueError(msg) + cfg['apt']['preserve_sources_list'] = False + + if cfg.get('debconf_selections') is not None: + if predef_apt_cfg.get('debconf_selections') is not None: + msg = ("Error in debconf_selections configuration: " + "old and new format of apt features " + "are mutually exclusive") + LOG.error(msg) + raise ValueError(msg) + + selsets = cfg.get('debconf_selections') + cfg['apt']['debconf_selections'] = selsets + LOG.info("Transferred %s into new format: %s", + cfg.get('debconf_selections'), + cfg.get('apt')) + del cfg['debconf_selections'] + + return cfg + + +CMD_ARGUMENTS = ( + ((('-c', '--config'), + {'help': 'read configuration from cfg', 'action': util.MergedCmdAppend, + 'metavar': 'FILE', 'type': argparse.FileType("rb"), + 'dest': 'cfgopts', 'default': []}), + (('-t', '--target'), + {'help': 'chroot to target. default is env[TARGET_MOUNT_POINT]', + 'action': 'store', 'metavar': 'TARGET', + 'default': os.environ.get('TARGET_MOUNT_POINT')}),) +) + + +def POPULATE_SUBCMD(parser): + """Populate subcommand option parsing for apt-config""" + populate_one_subcmd(parser, CMD_ARGUMENTS, apt_command) + +CONFIG_CLEANERS = { + 'cloud-init': clean_cloud_init, +} + +# vi: ts=4 expandtab syntax=python === added file 'curtin/commands/block_info.py' --- curtin/commands/block_info.py 1970-01-01 00:00:00 +0000 +++ curtin/commands/block_info.py 2016-10-03 18:55:20 +0000 @@ -0,0 +1,75 @@ +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Wesley Wiedenmeier +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +import os +from . import populate_one_subcmd +from curtin import (block, util) + + +def block_info_main(args): + """get information about block devices, similar to lsblk""" + if not args.devices: + raise ValueError('devices to scan must be specified') + if not all(block.is_block_device(d) for d in args.devices): + raise ValueError('invalid device(s)') + + def add_size_to_holders_tree(tree): + """add size information to generated holders trees""" + size_file = os.path.join(tree['device'], 'size') + # size file is always represented in 512 byte sectors even if + # underlying disk uses a larger logical_block_size + size = ((512 * int(util.load_file(size_file))) + if os.path.exists(size_file) else None) + tree['size'] = util.bytes2human(size) if args.human else str(size) + for holder in tree['holders']: + add_size_to_holders_tree(holder) + return tree + + def format_name(tree): + """format information for human readable display""" + res = { + 'name': ' - '.join((tree['name'], tree['dev_type'], tree['size'])), + 'holders': [] + } + for holder in tree['holders']: + res['holders'].append(format_name(holder)) + return res + + trees = [add_size_to_holders_tree(t) for t in + [block.clear_holders.gen_holders_tree(d) for d in args.devices]] + + print(util.json_dumps(trees) if args.json else + '\n'.join(block.clear_holders.format_holders_tree(t) for t in + [format_name(tree) for tree in trees])) + + return 0 + + +CMD_ARGUMENTS = ( + ('devices', + {'help': 'devices to get info for', 'default': [], 'nargs': '+'}), + ('--human', + {'help': 'output size in human readable format', 'default': False, + 'action': 'store_true'}), + (('-j', '--json'), + {'help': 'output data in json format', 'default': False, + 'action': 'store_true'}), +) + + +def POPULATE_SUBCMD(parser): + populate_one_subcmd(parser, CMD_ARGUMENTS, block_info_main) === modified file 'curtin/commands/block_meta.py' --- curtin/commands/block_meta.py 2016-10-03 18:00:41 +0000 +++ curtin/commands/block_meta.py 2016-10-03 18:55:20 +0000 @@ -17,9 +17,8 @@ from collections import OrderedDict from curtin import (block, config, util) -from curtin.block import mdadm +from curtin.block import (mdadm, mkfs, clear_holders, lvm) from curtin.log import LOG -from curtin.block import mkfs from curtin.reporter import events from . import populate_one_subcmd @@ -28,7 +27,7 @@ import glob import os import platform -import re +import string import sys import tempfile import time @@ -129,128 +128,6 @@ return "mbr" -def block_find_sysfs_path(devname): - # return the path in sys for device named devname - # support either short name ('sda') or full path /dev/sda - # sda -> /sys/class/block/sda - # sda1 -> /sys/class/block/sda/sda1 - if not devname: - raise ValueError("empty devname provided to find_sysfs_path") - - sys_class_block = '/sys/class/block/' - basename = os.path.basename(devname) - # try without parent blockdevice, then prepend parent - paths = [ - os.path.join(sys_class_block, basename), - os.path.join(sys_class_block, - re.split('[\d+]', basename)[0], basename), - ] - - # find path to devname directory in sysfs - devname_sysfs = None - for path in paths: - if os.path.exists(path): - devname_sysfs = path - - if devname_sysfs is None: - err = ('No sysfs path to device:' - ' {}'.format(devname_sysfs)) - LOG.error(err) - raise ValueError(err) - - return devname_sysfs - - -def get_holders(devname): - # Look up any block device holders. - # Handle devices and partitions as devnames (vdb, md0, vdb7) - devname_sysfs = block_find_sysfs_path(devname) - if devname_sysfs: - holders = os.listdir(os.path.join(devname_sysfs, 'holders')) - LOG.debug("devname '%s' had holders: %s", devname, ','.join(holders)) - return holders - - LOG.debug('get_holders: did not find sysfs path for %s', devname) - return [] - - -def clear_holders(sys_block_path): - holders = os.listdir(os.path.join(sys_block_path, "holders")) - LOG.info("clear_holders running on '%s', with holders '%s'" % - (sys_block_path, holders)) - for holder in holders: - # get path to holder in /sys/block, then clear it - try: - holder_realpath = os.path.realpath( - os.path.join(sys_block_path, "holders", holder)) - clear_holders(holder_realpath) - except IOError as e: - # something might have already caused the holder to go away - if util.is_file_not_found_exc(e): - pass - pass - - # detect what type of holder is using this volume and shut it down, need to - # find more robust name of doing detection - if "bcache" in sys_block_path: - # bcache device - part_devs = [] - for part_dev in glob.glob(os.path.join(sys_block_path, - "slaves", "*", "dev")): - with open(part_dev, "r") as fp: - part_dev_id = fp.read().rstrip() - part_devs.append( - os.path.split(os.path.realpath(os.path.join("/dev/block", - part_dev_id)))[-1]) - for cache_dev in glob.glob("/sys/fs/bcache/*/bdev*"): - for part_dev in part_devs: - if part_dev in os.path.realpath(cache_dev): - # This is our bcache device, stop it, wait for udev to - # settle - with open(os.path.join(os.path.split(cache_dev)[0], - "stop"), "w") as fp: - LOG.info("stopping: %s" % fp) - fp.write("1") - udevadm_settle() - break - for part_dev in part_devs: - block.wipe_volume(os.path.join("/dev", part_dev), - mode="superblock") - - if os.path.exists(os.path.join(sys_block_path, "bcache")): - # bcache device that isn't running, if it were, we would have found it - # when we looked for holders - try: - with open(os.path.join(sys_block_path, "bcache", "set", "stop"), - "w") as fp: - LOG.info("stopping: %s" % fp) - fp.write("1") - except IOError as e: - if not util.is_file_not_found_exc(e): - raise e - with open(os.path.join(sys_block_path, "bcache", "stop"), - "w") as fp: - LOG.info("stopping: %s" % fp) - fp.write("1") - udevadm_settle() - - if os.path.exists(os.path.join(sys_block_path, "md")): - # md device - block_dev = os.path.join("/dev/", os.path.split(sys_block_path)[-1]) - # if these fail its okay, the array might not be assembled and thats - # fine - mdadm.mdadm_stop(block_dev) - mdadm.mdadm_remove(block_dev) - - elif os.path.exists(os.path.join(sys_block_path, "dm")): - # Shut down any volgroups - with open(os.path.join(sys_block_path, "dm", "name"), "r") as fp: - name = fp.read().split('-') - util.subp(["lvremove", "--force", name[0].rstrip(), name[1].rstrip()], - rcs=[0, 5]) - util.subp(["vgremove", name[0].rstrip()], rcs=[0, 5, 6]) - - def devsync(devpath): LOG.debug('devsync for %s', devpath) util.subp(['partprobe', devpath], rcs=[0, 1]) @@ -265,14 +142,6 @@ raise OSError('Failed to find device at path: %s', devpath) -def determine_partition_kname(disk_kname, partition_number): - for dev_type in ["nvme", "mmcblk"]: - if disk_kname.startswith(dev_type): - partition_number = "p%s" % partition_number - break - return "%s%s" % (disk_kname, partition_number) - - def determine_partition_number(partition_id, storage_config): vol = storage_config.get(partition_id) partnumber = vol.get('number') @@ -304,6 +173,18 @@ return partnumber +def sanitize_dname(dname): + """ + dnames should be sanitized before writing rule files, in case maas has + emitted a dname with a special character + + only letters, numbers and '-' and '_' are permitted, as this will be + used for a device path. spaces are also not permitted + """ + valid = string.digits + string.ascii_letters + '-_' + return ''.join(c if c in valid else '-' for c in dname) + + def make_dname(volume, storage_config): state = util.load_command_environment() rules_dir = os.path.join(state['scratch'], "rules.d") @@ -321,7 +202,7 @@ # we may not always be able to find a uniq identifier on devices with names if not ptuuid and vol.get('type') in ["disk", "partition"]: LOG.warning("Can't find a uuid for volume: {}. Skipping dname.".format( - dname)) + volume)) return rule = [ @@ -346,11 +227,24 @@ volgroup_name = storage_config.get(vol.get('volgroup')).get('name') dname = "%s-%s" % (volgroup_name, dname) rule.append(compose_udev_equality("ENV{DM_NAME}", dname)) - rule.append("SYMLINK+=\"disk/by-dname/%s\"" % dname) + else: + raise ValueError('cannot make dname for device with type: {}' + .format(vol.get('type'))) + + # note: this sanitization is done here instead of for all name attributes + # at the beginning of storage configuration, as some devices, such as + # lvm devices may use the name attribute and may permit special chars + sanitized = sanitize_dname(dname) + if sanitized != dname: + LOG.warning( + "dname modified to remove invalid chars. old: '{}' new: '{}'" + .format(dname, sanitized)) + + rule.append("SYMLINK+=\"disk/by-dname/%s\"" % sanitized) LOG.debug("Writing dname udev rule '{}'".format(str(rule))) util.ensure_dir(rules_dir) - with open(os.path.join(rules_dir, volume), "w") as fp: - fp.write(', '.join(rule)) + rule_file = os.path.join(rules_dir, '{}.rules'.format(sanitized)) + util.write_file(rule_file, ', '.join(rule)) def get_path_to_storage_volume(volume, storage_config): @@ -368,9 +262,9 @@ partnumber = determine_partition_number(vol.get('id'), storage_config) disk_block_path = get_path_to_storage_volume(vol.get('device'), storage_config) - (base_path, disk_kname) = os.path.split(disk_block_path) - partition_kname = determine_partition_kname(disk_kname, partnumber) - volume_path = os.path.join(base_path, partition_kname) + disk_kname = block.path_to_kname(disk_block_path) + partition_kname = block.partition_kname(disk_kname, partnumber) + volume_path = block.kname_to_path(partition_kname) devsync_vol = os.path.join(disk_block_path) elif vol.get('type') == "disk": @@ -419,13 +313,15 @@ # block devs are in the slaves dir there. Then, those blockdevs can be # checked against the kname of the devs in the config for the desired # bcache device. This is not very elegant though - backing_device_kname = os.path.split(get_path_to_storage_volume( - vol.get('backing_device'), storage_config))[-1] + backing_device_path = get_path_to_storage_volume( + vol.get('backing_device'), storage_config) + backing_device_kname = block.path_to_kname(backing_device_path) sys_path = list(filter(lambda x: backing_device_kname in x, glob.glob("/sys/block/bcache*/slaves/*")))[0] while "bcache" not in os.path.split(sys_path)[-1]: sys_path = os.path.split(sys_path)[0] - volume_path = os.path.join("/dev", os.path.split(sys_path)[-1]) + bcache_kname = block.path_to_kname(sys_path) + volume_path = block.kname_to_path(bcache_kname) LOG.debug('got bcache volume path {}'.format(volume_path)) else: @@ -442,62 +338,35 @@ def disk_handler(info, storage_config): + _dos_names = ['dos', 'msdos'] ptable = info.get('ptable') - disk = get_path_to_storage_volume(info.get('id'), storage_config) - # Handle preserve flag - if info.get('preserve'): - if not ptable: - # Don't need to check state, return - return - - # Check state of current ptable - try: - (out, _err) = util.subp(["blkid", "-o", "export", disk], - capture=True) - except util.ProcessExecutionError: - raise ValueError("disk '%s' has no readable partition table or \ - cannot be accessed, but preserve is set to true, so cannot \ - continue") - current_ptable = list(filter(lambda x: "PTTYPE" in x, - out.splitlines()))[0].split("=")[-1] - if current_ptable == "dos" and ptable != "msdos" or \ - current_ptable == "gpt" and ptable != "gpt": - raise ValueError("disk '%s' does not have correct \ - partition table, but preserve is set to true, so not \ - creating table, so not creating table." % info.get('id')) - LOG.info("disk '%s' marked to be preserved, so keeping partition \ - table") - return - - # Wipe the disk - if info.get('wipe') and info.get('wipe') != "none": - # The disk has a lable, clear all partitions - mdadm.mdadm_assemble(scan=True) - disk_kname = os.path.split(disk)[-1] - syspath_partitions = list( - os.path.split(prt)[0] for prt in - glob.glob("/sys/block/%s/*/partition" % disk_kname)) - for partition in syspath_partitions: - clear_holders(partition) - with open(os.path.join(partition, "dev"), "r") as fp: - block_no = fp.read().rstrip() - partition_path = os.path.realpath( - os.path.join("/dev/block", block_no)) - block.wipe_volume(partition_path, mode=info.get('wipe')) - - clear_holders("/sys/block/%s" % disk_kname) - block.wipe_volume(disk, mode=info.get('wipe')) - - # Create partition table on disk - if info.get('ptable'): - LOG.info("labeling device: '%s' with '%s' partition table", disk, - ptable) - if ptable == "gpt": - util.subp(["sgdisk", "--clear", disk]) - elif ptable == "msdos": - util.subp(["parted", disk, "--script", "mklabel", "msdos"]) + if config.value_as_boolean(info.get('preserve')): + # Handle preserve flag, verifying if ptable specified in config + if config.value_as_boolean(ptable): + current_ptable = block.get_part_table_type(disk) + if not ((ptable in _dos_names and current_ptable in _dos_names) or + (ptable == 'gpt' and current_ptable == 'gpt')): + raise ValueError( + "disk '%s' does not have correct partition table or " + "cannot be read, but preserve is set to true. " + "cannot continue installation." % info.get('id')) + LOG.info("disk '%s' marked to be preserved, so keeping partition " + "table" % disk) + else: + # wipe the disk and create the partition table if instructed to do so + if config.value_as_boolean(info.get('wipe')): + block.wipe_volume(disk, mode=info.get('wipe')) + if config.value_as_boolean(ptable): + LOG.info("labeling device: '%s' with '%s' partition table", disk, + ptable) + if ptable == "gpt": + util.subp(["sgdisk", "--clear", disk]) + elif ptable in _dos_names: + util.subp(["parted", disk, "--script", "mklabel", "msdos"]) + else: + raise ValueError('invalid partition table type: %s', ptable) # Make the name if needed if info.get('name'): @@ -542,13 +411,12 @@ disk = get_path_to_storage_volume(device, storage_config) partnumber = determine_partition_number(info.get('id'), storage_config) - - disk_kname = os.path.split( - get_path_to_storage_volume(device, storage_config))[-1] + disk_kname = block.path_to_kname(disk) + disk_sysfs_path = block.sys_block_path(disk) # consider the disks logical sector size when calculating sectors try: - prefix = "/sys/block/%s/queue/" % disk_kname - with open(prefix + "logical_block_size", "r") as f: + lbs_path = os.path.join(disk_sysfs_path, 'queue', 'logical_block_size') + with open(lbs_path, 'r') as f: l = f.readline() logical_block_size_bytes = int(l) except: @@ -566,17 +434,14 @@ extended_part_no = determine_partition_number( key, storage_config) break - partition_kname = determine_partition_kname( - disk_kname, extended_part_no) - previous_partition = "/sys/block/%s/%s/" % \ - (disk_kname, partition_kname) + pnum = extended_part_no else: pnum = find_previous_partition(device, info['id'], storage_config) - LOG.debug("previous partition number for '%s' found to be '%s'", - info.get('id'), pnum) - partition_kname = determine_partition_kname(disk_kname, pnum) - previous_partition = "/sys/block/%s/%s/" % \ - (disk_kname, partition_kname) + + LOG.debug("previous partition number for '%s' found to be '%s'", + info.get('id'), pnum) + partition_kname = block.partition_kname(disk_kname, pnum) + previous_partition = os.path.join(disk_sysfs_path, partition_kname) LOG.debug("previous partition: {}".format(previous_partition)) # XXX: sys/block/X/{size,start} is *ALWAYS* in 512b value previous_size = util.load_file(os.path.join(previous_partition, @@ -629,9 +494,9 @@ length_sectors = length_sectors + (logdisks * alignment_offset) # Handle preserve flag - if info.get('preserve'): + if config.value_as_boolean(info.get('preserve')): return - elif storage_config.get(device).get('preserve'): + elif config.value_as_boolean(storage_config.get(device).get('preserve')): raise NotImplementedError("Partition '%s' is not marked to be \ preserved, but device '%s' is. At this time, preserving devices \ but not also the partitions on the devices is not supported, \ @@ -674,11 +539,16 @@ else: raise ValueError("parent partition has invalid partition table") - # Wipe the partition if told to do so - if info.get('wipe') and info.get('wipe') != "none": - block.wipe_volume( - get_path_to_storage_volume(info.get('id'), storage_config), - mode=info.get('wipe')) + # Wipe the partition if told to do so, do not wipe dos extended partitions + # as this may damage the extended partition table + if config.value_as_boolean(info.get('wipe')): + if info.get('flag') == "extended": + LOG.warn("extended partitions do not need wiping, so skipping: " + "'%s'" % info.get('id')) + else: + block.wipe_volume( + get_path_to_storage_volume(info.get('id'), storage_config), + mode=info.get('wipe')) # Make the name if needed if storage_config.get(device).get('name') and partition_type != 'extended': make_dname(info.get('id'), storage_config) @@ -694,7 +564,7 @@ volume_path = get_path_to_storage_volume(volume, storage_config) # Handle preserve flag - if info.get('preserve'): + if config.value_as_boolean(info.get('preserve')): # Volume marked to be preserved, not formatting return @@ -776,26 +646,21 @@ storage_config)) # Handle preserve flag - if info.get('preserve'): + if config.value_as_boolean(info.get('preserve')): # LVM will probably be offline, so start it util.subp(["vgchange", "-a", "y"]) # Verify that volgroup exists and contains all specified devices - current_paths = [] - (out, _err) = util.subp(["pvdisplay", "-C", "--separator", "=", "-o", - "vg_name,pv_name", "--noheadings"], - capture=True) - for line in out.splitlines(): - if name in line: - current_paths.append(line.split("=")[-1]) - if set(current_paths) != set(device_paths): - raise ValueError("volgroup '%s' marked to be preserved, but does \ - not exist or does not contain the right physical \ - volumes" % info.get('id')) + if set(lvm.get_pvols_in_volgroup(name)) != set(device_paths): + raise ValueError("volgroup '%s' marked to be preserved, but does " + "not exist or does not contain the right " + "physical volumes" % info.get('id')) else: # Create vgrcreate command and run - cmd = ["vgcreate", name] - cmd.extend(device_paths) - util.subp(cmd) + # capture output to avoid printing it to log + util.subp(['vgcreate', name] + device_paths, capture=True) + + # refresh lvmetad + lvm.lvm_scan() def lvm_partition_handler(info, storage_config): @@ -805,28 +670,23 @@ raise ValueError("lvm volgroup for lvm partition must be specified") if not name: raise ValueError("lvm partition name must be specified") + if info.get('ptable'): + raise ValueError("Partition tables on top of lvm logical volumes is " + "not supported") # Handle preserve flag - if info.get('preserve'): - (out, _err) = util.subp(["lvdisplay", "-C", "--separator", "=", "-o", - "lv_name,vg_name", "--noheadings"], - capture=True) - found = False - for line in out.splitlines(): - if name in line: - if volgroup == line.split("=")[-1]: - found = True - break - if not found: - raise ValueError("lvm partition '%s' marked to be preserved, but \ - does not exist or does not mach storage \ - configuration" % info.get('id')) + if config.value_as_boolean(info.get('preserve')): + if name not in lvm.get_lvols_in_volgroup(volgroup): + raise ValueError("lvm partition '%s' marked to be preserved, but " + "does not exist or does not mach storage " + "configuration" % info.get('id')) elif storage_config.get(info.get('volgroup')).get('preserve'): - raise NotImplementedError("Lvm Partition '%s' is not marked to be \ - preserved, but volgroup '%s' is. At this time, preserving \ - volgroups but not also the lvm partitions on the volgroup is \ - not supported, because of the possibility of damaging lvm \ - partitions intended to be preserved." % (info.get('id'), volgroup)) + raise NotImplementedError( + "Lvm Partition '%s' is not marked to be preserved, but volgroup " + "'%s' is. At this time, preserving volgroups but not also the lvm " + "partitions on the volgroup is not supported, because of the " + "possibility of damaging lvm partitions intended to be " + "preserved." % (info.get('id'), volgroup)) else: cmd = ["lvcreate", volgroup, "-n", name] if info.get('size'): @@ -836,9 +696,8 @@ util.subp(cmd) - if info.get('ptable'): - raise ValueError("Partition tables on top of lvm logical volumes is \ - not supported") + # refresh lvmetad + lvm.lvm_scan() make_dname(info.get('id'), storage_config) @@ -925,7 +784,7 @@ zip(spare_devices, spare_device_paths))) # Handle preserve flag - if info.get('preserve'): + if config.value_as_boolean(info.get('preserve')): # check if the array is already up, if not try to assemble if not mdadm.md_check(md_devname, raidlevel, device_paths, spare_device_paths): @@ -981,9 +840,6 @@ raise ValueError("backing device and cache device for bcache" " must be specified") - # The bcache module is not loaded when bcache is installed by apt-get, so - # we will load it now - util.subp(["modprobe", "bcache"]) bcache_sysfs = "/sys/fs/bcache" udevadm_settle(exists=bcache_sysfs) @@ -1003,7 +859,7 @@ bcache_device, expected) return LOG.debug('bcache device path not found: %s', expected) - local_holders = get_holders(bcache_device) + local_holders = clear_holders.get_holders(bcache_device) LOG.debug('got initial holders being "%s"', local_holders) if len(local_holders) == 0: raise ValueError("holders == 0 , expected non-zero") @@ -1033,7 +889,7 @@ if cache_device: # /sys/class/block/XXX/YYY/ - cache_device_sysfs = block_find_sysfs_path(cache_device) + cache_device_sysfs = block.sys_block_path(cache_device) if os.path.exists(os.path.join(cache_device_sysfs, "bcache")): LOG.debug('caching device already exists at {}/bcache. Read ' @@ -1058,7 +914,7 @@ ensure_bcache_is_registered(cache_device, target_sysfs_path) if backing_device: - backing_device_sysfs = block_find_sysfs_path(backing_device) + backing_device_sysfs = block.sys_block_path(backing_device) target_sysfs_path = os.path.join(backing_device_sysfs, "bcache") if not os.path.exists(os.path.join(backing_device_sysfs, "bcache")): util.subp(["make-bcache", "-B", backing_device]) @@ -1066,7 +922,7 @@ # via the holders we can identify which bcache device we just created # for a given backing device - holders = get_holders(backing_device) + holders = clear_holders.get_holders(backing_device) if len(holders) != 1: err = ('Invalid number {} of holding devices:' ' "{}"'.format(len(holders), holders)) @@ -1158,6 +1014,21 @@ # set up reportstack stack_prefix = state.get('report_stack_prefix', '') + # shut down any already existing storage layers above any disks used in + # config that have 'wipe' set + with events.ReportEventStack( + name=stack_prefix, reporting_enabled=True, level='INFO', + description="removing previous storage devices"): + clear_holders.start_clear_holders_deps() + disk_paths = [get_path_to_storage_volume(k, storage_config_dict) + for (k, v) in storage_config_dict.items() + if v.get('type') == 'disk' and + config.value_as_boolean(v.get('wipe')) and + not config.value_as_boolean(v.get('preserve'))] + clear_holders.clear_holders(disk_paths) + # if anything was not properly shut down, stop installation + clear_holders.assert_clear(disk_paths) + for item_id, command in storage_config_dict.items(): handler = command_handlers.get(command['type']) if not handler: === modified file 'curtin/commands/block_wipe.py' --- curtin/commands/block_wipe.py 2016-05-10 16:13:29 +0000 +++ curtin/commands/block_wipe.py 2016-10-03 18:55:20 +0000 @@ -21,7 +21,6 @@ def wipe_main(args): - # curtin clear-holders device [device2 [device3]] for blockdev in args.devices: try: block.wipe_volume(blockdev, mode=args.mode) @@ -36,7 +35,7 @@ CMD_ARGUMENTS = ( ((('-m', '--mode'), {'help': 'mode for wipe.', 'action': 'store', - 'default': 'superblocks', + 'default': 'superblock', 'choices': ['zero', 'superblock', 'superblock-recursive', 'random']}), ('devices', {'help': 'devices to wipe', 'default': [], 'nargs': '+'}), === added file 'curtin/commands/clear_holders.py' --- curtin/commands/clear_holders.py 1970-01-01 00:00:00 +0000 +++ curtin/commands/clear_holders.py 2016-10-03 18:55:20 +0000 @@ -0,0 +1,48 @@ +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Wesley Wiedenmeier +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . + +from curtin import block +from . import populate_one_subcmd + + +def clear_holders_main(args): + """ + wrapper for clear_holders accepting cli args + """ + if (not all(block.is_block_device(device) for device in args.devices) or + len(args.devices) == 0): + raise ValueError('invalid devices specified') + block.clear_holders.start_clear_holders_deps() + block.clear_holders.clear_holders(args.devices, try_preserve=args.preserve) + if args.try_preserve: + print('ran clear_holders attempting to preserve data. however, ' + 'hotplug support for some devices may cause holders to restart ') + block.clear_holders.assert_clear(args.devices) + + +CMD_ARGUMENTS = ( + (('devices', + {'help': 'devices to free', 'default': [], 'nargs': '+'}), + (('-p', '--preserve'), + {'help': 'try to shut down holders without erasing anything', + 'default': False, 'action': 'store_true'}), + ) +) + + +def POPULATE_SUBCMD(parser): + populate_one_subcmd(parser, CMD_ARGUMENTS, clear_holders_main) === modified file 'curtin/commands/curthooks.py' --- curtin/commands/curthooks.py 2016-10-03 18:00:41 +0000 +++ curtin/commands/curthooks.py 2016-10-03 18:55:20 +0000 @@ -16,10 +16,8 @@ # along with Curtin. If not, see . import copy -import glob import os import platform -import re import sys import shutil import textwrap @@ -30,8 +28,8 @@ from curtin.log import LOG from curtin import swap from curtin import util -from curtin import net from curtin.reporter import events +from curtin.commands import apply_net, apt_config from . import populate_one_subcmd @@ -90,45 +88,15 @@ info.get('perms', "0644"))) -def apt_config(cfg, target): - # cfg['apt_proxy'] - - proxy_cfg_path = os.path.sep.join( - [target, '/etc/apt/apt.conf.d/90curtin-aptproxy']) - if cfg.get('apt_proxy'): - util.write_file( - proxy_cfg_path, - content='Acquire::HTTP::Proxy "%s";\n' % cfg['apt_proxy']) +def do_apt_config(cfg, target): + cfg = apt_config.translate_old_apt_features(cfg) + apt_cfg = cfg.get("apt") + if apt_cfg is not None: + LOG.info("curthooks handling apt to target %s with config %s", + target, apt_cfg) + apt_config.handle_apt(apt_cfg, target) else: - if os.path.isfile(proxy_cfg_path): - os.unlink(proxy_cfg_path) - - # cfg['apt_mirrors'] - # apt_mirrors: - # ubuntu_archive: http://local.archive/ubuntu - # ubuntu_security: http://local.archive/ubuntu - sources_list = os.path.sep.join([target, '/etc/apt/sources.list']) - if (isinstance(cfg.get('apt_mirrors'), dict) and - os.path.isfile(sources_list)): - repls = [ - ('ubuntu_archive', r'http://\S*[.]*archive.ubuntu.com/\S*'), - ('ubuntu_security', r'http://security.ubuntu.com/\S*'), - ] - content = None - for name, regex in repls: - mirror = cfg['apt_mirrors'].get(name) - if not mirror: - continue - - if content is None: - with open(sources_list) as fp: - content = fp.read() - util.write_file(sources_list + ".dist", content) - - content = re.sub(regex, mirror + " ", content) - - if content is not None: - util.write_file(sources_list, content) + LOG.info("No apt config provided, skipping") def disable_overlayroot(cfg, target): @@ -140,51 +108,6 @@ shutil.move(local_conf, local_conf + ".old") -def clean_cloud_init(target): - flist = glob.glob( - os.path.sep.join([target, "/etc/cloud/cloud.cfg.d/*dpkg*"])) - - LOG.debug("cleaning cloud-init config from: %s" % flist) - for dpkg_cfg in flist: - os.unlink(dpkg_cfg) - - -def _maybe_remove_legacy_eth0(target, - path="/etc/network/interfaces.d/eth0.cfg"): - """Ubuntu cloud images previously included a 'eth0.cfg' that had - hard coded content. That file would interfere with the rendered - configuration if it was present. - - if the file does not exist do nothing. - If the file exists: - - with known content, remove it and warn - - with unknown content, leave it and warn - """ - - cfg = os.path.sep.join([target, path]) - if not os.path.exists(cfg): - LOG.warn('Failed to find legacy conf file %s', cfg) - return - - bmsg = "Dynamic networking config may not apply." - try: - contents = util.load_file(cfg) - known_contents = ["auto eth0", "iface eth0 inet dhcp"] - lines = [f.strip() for f in contents.splitlines() - if not f.startswith("#")] - if lines == known_contents: - util.del_file(cfg) - msg = "removed %s with known contents" % cfg - else: - msg = (bmsg + " '%s' exists with user configured content." % cfg) - except: - msg = bmsg + " %s exists, but could not be read." % cfg - LOG.exception(msg) - return - - LOG.warn(msg) - - def setup_zipl(cfg, target): if platform.machine() != 's390x': return @@ -232,8 +155,8 @@ def run_zipl(cfg, target): if platform.machine() != 's390x': return - with util.RunInChroot(target) as in_chroot: - in_chroot(['zipl']) + with util.ChrootableTarget(target) as in_chroot: + in_chroot.subp(['zipl']) def install_kernel(cfg, target): @@ -250,126 +173,45 @@ mapping = copy.deepcopy(KERNEL_MAPPING) config.merge_config(mapping, kernel_cfg.get('mapping', {})) - with util.RunInChroot(target) as in_chroot: - - if kernel_package: - util.install_packages([kernel_package], target=target) - return - - # uname[2] is kernel name (ie: 3.16.0-7-generic) - # version gets X.Y.Z, flavor gets anything after second '-'. - kernel = os.uname()[2] - codename, err = in_chroot(['lsb_release', '--codename', '--short'], - capture=True) - codename = codename.strip() - version, abi, flavor = kernel.split('-', 2) - - try: - map_suffix = mapping[codename][version] - except KeyError: - LOG.warn("Couldn't detect kernel package to install for %s." - % kernel) - if kernel_fallback is not None: - util.install_packages([kernel_fallback], target=target) - return - - package = "linux-{flavor}{map_suffix}".format( - flavor=flavor, map_suffix=map_suffix) - - if util.has_pkg_available(package, target): - if util.has_pkg_installed(package, target): - LOG.debug("Kernel package '%s' already installed", package) - else: - LOG.debug("installing kernel package '%s'", package) - util.install_packages([package], target=target) - else: - if kernel_fallback is not None: - LOG.info("Kernel package '%s' not available. " - "Installing fallback package '%s'.", - package, kernel_fallback) - util.install_packages([kernel_fallback], target=target) - else: - LOG.warn("Kernel package '%s' not available and no fallback." - " System may not boot.", package) - - -def apply_debconf_selections(cfg, target): - # debconf_selections: - # set1: | - # cloud-init cloud-init/datasources multiselect MAAS - # set2: pkg pkg/value string bar - selsets = cfg.get('debconf_selections') - if not selsets: - LOG.debug("debconf_selections was not set in config") - return - - # for each entry in selections, chroot and apply them. - # keep a running total of packages we've seen. - pkgs_cfgd = set() - for key, content in selsets.items(): - LOG.debug("setting for %s, %s" % (key, content)) - util.subp(['chroot', target, 'debconf-set-selections'], - data=content.encode()) - for line in content.splitlines(): - if line.startswith("#"): - continue - pkg = re.sub(r"[:\s].*", "", line) - pkgs_cfgd.add(pkg) - - pkgs_installed = get_installed_packages(target) - - LOG.debug("pkgs_cfgd: %s" % pkgs_cfgd) - LOG.debug("pkgs_installed: %s" % pkgs_installed) - need_reconfig = pkgs_cfgd.intersection(pkgs_installed) - - if len(need_reconfig) == 0: - LOG.debug("no need for reconfig") - return - - # For any packages that are already installed, but have preseed data - # we populate the debconf database, but the filesystem configuration - # would be preferred on a subsequent dpkg-reconfigure. - # so, what we have to do is "know" information about certain packages - # to unconfigure them. - unhandled = [] - to_config = [] - for pkg in need_reconfig: - if pkg in CONFIG_CLEANERS: - LOG.debug("unconfiguring %s" % pkg) - CONFIG_CLEANERS[pkg](target) - to_config.append(pkg) - else: - unhandled.append(pkg) - - if len(unhandled): - LOG.warn("The following packages were installed and preseeded, " - "but cannot be unconfigured: %s", unhandled) - - util.subp(['chroot', target, 'dpkg-reconfigure', - '--frontend=noninteractive'] + - list(to_config), data=None) - - -def get_installed_packages(target=None): - cmd = [] - if target is not None: - cmd = ['chroot', target] - cmd.extend(['dpkg-query', '--list']) - - (out, _err) = util.subp(cmd, capture=True) - if isinstance(out, bytes): - out = out.decode() - - pkgs_inst = set() - for line in out.splitlines(): - try: - (state, pkg, other) = line.split(None, 2) - except ValueError: - continue - if state.startswith("hi") or state.startswith("ii"): - pkgs_inst.add(re.sub(":.*", "", pkg)) - - return pkgs_inst + if kernel_package: + util.install_packages([kernel_package], target=target) + return + + # uname[2] is kernel name (ie: 3.16.0-7-generic) + # version gets X.Y.Z, flavor gets anything after second '-'. + kernel = os.uname()[2] + codename, _ = util.subp(['lsb_release', '--codename', '--short'], + capture=True, target=target) + codename = codename.strip() + version, abi, flavor = kernel.split('-', 2) + + try: + map_suffix = mapping[codename][version] + except KeyError: + LOG.warn("Couldn't detect kernel package to install for %s." + % kernel) + if kernel_fallback is not None: + util.install_packages([kernel_fallback], target=target) + return + + package = "linux-{flavor}{map_suffix}".format( + flavor=flavor, map_suffix=map_suffix) + + if util.has_pkg_available(package, target): + if util.has_pkg_installed(package, target): + LOG.debug("Kernel package '%s' already installed", package) + else: + LOG.debug("installing kernel package '%s'", package) + util.install_packages([package], target=target) + else: + if kernel_fallback is not None: + LOG.info("Kernel package '%s' not available. " + "Installing fallback package '%s'.", + package, kernel_fallback) + util.install_packages([kernel_fallback], target=target) + else: + LOG.warn("Kernel package '%s' not available and no fallback." + " System may not boot.", package) def setup_grub(cfg, target): @@ -498,12 +340,11 @@ util.subp(args + instdevs, env=env) -def update_initramfs(target, all_kernels=False): +def update_initramfs(target=None, all_kernels=False): cmd = ['update-initramfs', '-u'] if all_kernels: cmd.extend(['-k', 'all']) - with util.RunInChroot(target) as in_chroot: - in_chroot(cmd) + util.subp(cmd, target=target) def copy_fstab(fstab, target): @@ -533,7 +374,6 @@ def apply_networking(target, state): - netstate = state.get('network_state') netconf = state.get('network_config') interfaces = state.get('interfaces') @@ -544,22 +384,13 @@ return True return False - ns = None - if is_valid_src(netstate): - LOG.debug("applying network_state") - ns = net.network_state.from_state_file(netstate) - elif is_valid_src(netconf): - LOG.debug("applying network_config") - ns = net.parse_net_config(netconf) - - if ns is not None: - net.render_network_state(target=target, network_state=ns) + if is_valid_src(netconf): + LOG.info("applying network_config") + apply_net.apply_net(target, network_state=None, network_config=netconf) else: LOG.debug("copying interfaces") copy_interfaces(interfaces, target) - _maybe_remove_legacy_eth0(target) - def copy_interfaces(interfaces, target): if not interfaces: @@ -704,8 +535,8 @@ # FIXME: this assumes grub. need more generic way to update root= util.ensure_dir(os.path.sep.join([target, os.path.dirname(grub_dev)])) - with util.RunInChroot(target) as in_chroot: - in_chroot(['update-grub']) + with util.ChrootableTarget(target) as in_chroot: + in_chroot.subp(['update-grub']) else: LOG.warn("Not sure how this will boot") @@ -740,7 +571,7 @@ } needed_packages = [] - installed_packages = get_installed_packages(target) + installed_packages = util.get_installed_packages(target) for cust_cfg, pkg_reqs in custom_configs.items(): if cust_cfg not in cfg: continue @@ -820,7 +651,7 @@ name=stack_prefix, reporting_enabled=True, level="INFO", description="writing config files and configuring apt"): write_files(cfg, target) - apt_config(cfg, target) + do_apt_config(cfg, target) disable_overlayroot(cfg, target) # packages may be needed prior to installing kernel @@ -834,8 +665,8 @@ copy_mdadm_conf(mdadm_location, target) # as per https://bugs.launchpad.net/ubuntu/+source/mdadm/+bug/964052 # reconfigure mdadm - util.subp(['chroot', target, 'dpkg-reconfigure', - '--frontend=noninteractive', 'mdadm'], data=None) + util.subp(['dpkg-reconfigure', '--frontend=noninteractive', 'mdadm'], + data=None, target=target) with events.ReportEventStack( name=stack_prefix, reporting_enabled=True, level="INFO", @@ -843,7 +674,6 @@ setup_zipl(cfg, target) install_kernel(cfg, target) run_zipl(cfg, target) - apply_debconf_selections(cfg, target) restore_dist_interfaces(cfg, target) @@ -906,8 +736,4 @@ populate_one_subcmd(parser, CMD_ARGUMENTS, curthooks) -CONFIG_CLEANERS = { - 'cloud-init': clean_cloud_init, -} - # vi: ts=4 expandtab syntax=python === modified file 'curtin/commands/main.py' --- curtin/commands/main.py 2016-05-10 16:13:29 +0000 +++ curtin/commands/main.py 2016-10-03 18:55:20 +0000 @@ -26,9 +26,10 @@ from ..deps import install_deps SUB_COMMAND_MODULES = [ - 'apply_net', 'block-meta', 'block-wipe', 'curthooks', 'extract', - 'hook', 'in-target', 'install', 'mkfs', 'net-meta', - 'pack', 'swap', 'system-install', 'system-upgrade'] + 'apply_net', 'block-info', 'block-meta', 'block-wipe', 'curthooks', + 'clear-holders', 'extract', 'hook', 'in-target', 'install', 'mkfs', + 'net-meta', 'apt-config', 'pack', 'swap', 'system-install', + 'system-upgrade'] def add_subcmd(subparser, subcmd): === modified file 'curtin/config.py' --- curtin/config.py 2016-03-18 14:16:45 +0000 +++ curtin/config.py 2016-10-03 18:55:20 +0000 @@ -138,6 +138,5 @@ def value_as_boolean(value): - if value in (False, None, '0', 0, 'False', 'false', ''): - return False - return True + false_values = (False, None, 0, '0', 'False', 'false', 'None', 'none', '') + return value not in false_values === added file 'curtin/gpg.py' --- curtin/gpg.py 1970-01-01 00:00:00 +0000 +++ curtin/gpg.py 2016-10-03 18:55:20 +0000 @@ -0,0 +1,74 @@ +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Scott Moser +# Christian Ehrhardt +# +# Curtin is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# Curtin is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for +# more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Curtin. If not, see . +""" gpg.py +gpg related utilities to get raw keys data by their id +""" + +from curtin import util + +from .log import LOG + + +def export_armour(key): + """Export gpg key, armoured key gets returned""" + try: + (armour, _) = util.subp(["gpg", "--export", "--armour", key], + capture=True) + except util.ProcessExecutionError as error: + # debug, since it happens for any key not on the system initially + LOG.debug('Failed to export armoured key "%s": %s', key, error) + armour = None + return armour + + +def recv_key(key, keyserver): + """Receive gpg key from the specified keyserver""" + LOG.debug('Receive gpg key "%s"', key) + try: + util.subp(["gpg", "--keyserver", keyserver, "--recv", key], + capture=True) + except util.ProcessExecutionError as error: + raise ValueError(('Failed to import key "%s" ' + 'from server "%s" - error %s') % + (key, keyserver, error)) + + +def delete_key(key): + """Delete the specified key from the local gpg ring""" + try: + util.subp(["gpg", "--batch", "--yes", "--delete-keys", key], + capture=True) + except util.ProcessExecutionError as error: + LOG.warn('Failed delete key "%s": %s', key, error) + + +def getkeybyid(keyid, keyserver='keyserver.ubuntu.com'): + """get gpg keyid from keyserver""" + armour = export_armour(keyid) + if not armour: + try: + recv_key(keyid, keyserver=keyserver) + armour = export_armour(keyid) + except ValueError: + LOG.exception('Failed to obtain gpg key %s', keyid) + raise + finally: + # delete just imported key to leave environment as it was before + delete_key(keyid) + + return armour === modified file 'curtin/net/__init__.py' --- curtin/net/__init__.py 2016-10-03 18:00:41 +0000 +++ curtin/net/__init__.py 2016-10-03 18:55:20 +0000 @@ -299,7 +299,7 @@ mac = iface.get('mac_address', '') # len(macaddr) == 2 * 6 + 5 == 17 if ifname and mac and len(mac) == 17: - content += generate_udev_rule(ifname, mac) + content += generate_udev_rule(ifname, mac.lower()) return content @@ -349,7 +349,7 @@ 'subnets', 'type', ] - if iface['type'] not in ['bond', 'bridge']: + if iface['type'] not in ['bond', 'bridge', 'vlan']: ignore_map.append('mac_address') for key, value in iface.items(): @@ -361,26 +361,52 @@ return content -def render_route(route): - content = "up route add" +def render_route(route, indent=""): + """When rendering routes for an iface, in some cases applying a route + may result in the route command returning non-zero which produces + some confusing output for users manually using ifup/ifdown[1]. To + that end, we will optionally include an '|| true' postfix to each + route line allowing users to work with ifup/ifdown without using + --force option. + + We may at somepoint not want to emit this additional postfix, and + add a 'strict' flag to this function. When called with strict=True, + then we will not append the postfix. + + 1. http://askubuntu.com/questions/168033/ + how-to-set-static-routes-in-ubuntu-server + """ + content = [] + up = indent + "post-up route add" + down = indent + "pre-down route del" + or_true = " || true" mapping = { 'network': '-net', 'netmask': 'netmask', 'gateway': 'gw', 'metric': 'metric', } - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - content += " %s %s" % (mapping[k], route[k]) - - content += '\n' - return content - - -def iface_start_entry(iface, index): + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + default_gw = " default gw %s" % route['gateway'] + content.append(up + default_gw + or_true) + content.append(down + default_gw + or_true) + elif route['network'] == '::' and route['netmask'] == 0: + # ipv6! + default_gw = " -A inet6 default gw %s" % route['gateway'] + content.append(up + default_gw + or_true) + content.append(down + default_gw + or_true) + else: + route_line = "" + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + route_line += " %s %s" % (mapping[k], route[k]) + content.append(up + route_line + or_true) + content.append(down + route_line + or_true) + return "\n".join(content) + + +def iface_start_entry(iface): fullname = iface['name'] - if index != 0: - fullname += ":%s" % index control = iface['control'] if control == "auto": @@ -397,6 +423,16 @@ "iface {fullname} {inet} {mode}\n").format(**subst) +def subnet_is_ipv6(subnet): + # 'static6' or 'dhcp6' + if subnet['type'].endswith('6'): + # This is a request for DHCPv6. + return True + elif subnet['type'] == 'static' and ":" in subnet['address']: + return True + return False + + def render_interfaces(network_state): ''' Given state, emit etc/network/interfaces content ''' @@ -424,42 +460,43 @@ content += "\n" subnets = iface.get('subnets', {}) if subnets: - for index, subnet in zip(range(0, len(subnets)), subnets): + for index, subnet in enumerate(subnets): if content[-2:] != "\n\n": content += "\n" iface['index'] = index iface['mode'] = subnet['type'] iface['control'] = subnet.get('control', 'auto') subnet_inet = 'inet' - if iface['mode'].endswith('6'): - # This is a request for DHCPv6. - subnet_inet += '6' - elif iface['mode'] == 'static' and ":" in subnet['address']: - # This is a static IPv6 address. + if subnet_is_ipv6(subnet): subnet_inet += '6' iface['inet'] = subnet_inet - if iface['mode'].startswith('dhcp'): + if subnet['type'].startswith('dhcp'): iface['mode'] = 'dhcp' - content += iface_start_entry(iface, index) + # do not emit multiple 'auto $IFACE' lines as older (precise) + # ifupdown complains + if "auto %s\n" % (iface['name']) in content: + iface['control'] = 'alias' + + content += iface_start_entry(iface) content += iface_add_subnet(iface, subnet) content += iface_add_attrs(iface, index) - if len(subnets) > 1 and index == 0: - for i in range(1, len(subnets)): - content += " post-up ifup %s:%s\n" % (iface['name'], - i) + + for route in subnet.get('routes', []): + content += render_route(route, indent=" ") + '\n' + else: # ifenslave docs say to auto the slave devices - if 'bond-master' in iface: + if 'bond-master' in iface or 'bond-slaves' in iface: content += "auto {name}\n".format(**iface) content += "iface {name} {inet} {mode}\n".format(**iface) - content += iface_add_attrs(iface, index) + content += iface_add_attrs(iface, 0) for route in network_state.get('routes'): content += render_route(route) # global replacements until v2 format - content = content.replace('mac_address', 'hwaddress') + content = content.replace('mac_address', 'hwaddress ether') # Play nice with others and source eni config files content += "\nsource /etc/network/interfaces.d/*.cfg\n" === modified file 'curtin/net/network_state.py' --- curtin/net/network_state.py 2015-10-02 16:19:07 +0000 +++ curtin/net/network_state.py 2016-10-03 18:55:20 +0000 @@ -121,6 +121,18 @@ iface = interfaces.get(command['name'], {}) for param, val in command.get('params', {}).items(): iface.update({param: val}) + + # convert subnet ipv6 netmask to cidr as needed + subnets = command.get('subnets') + if subnets: + for subnet in subnets: + if subnet['type'] == 'static': + if 'netmask' in subnet and ':' in subnet['address']: + subnet['netmask'] = mask2cidr(subnet['netmask']) + for route in subnet.get('routes', []): + if 'netmask' in route: + route['netmask'] = mask2cidr(route['netmask']) + iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -130,7 +142,7 @@ 'mtu': command.get('mtu'), 'address': None, 'gateway': None, - 'subnets': command.get('subnets'), + 'subnets': subnets, }) self.network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() @@ -141,6 +153,7 @@ iface eth0.222 inet static address 10.10.10.1 netmask 255.255.255.0 + hwaddress ether BC:76:4E:06:96:B3 vlan-raw-device eth0 ''' required_keys = [ @@ -332,6 +345,37 @@ return ".".join([str(x) for x in mask]) +def ipv4mask2cidr(mask): + if '.' not in mask: + return mask + return sum([bin(int(x)).count('1') for x in mask.split('.')]) + + +def ipv6mask2cidr(mask): + if ':' not in mask: + return mask + + bitCount = [0, 0x8000, 0xc000, 0xe000, 0xf000, 0xf800, 0xfc00, 0xfe00, + 0xff00, 0xff80, 0xffc0, 0xffe0, 0xfff0, 0xfff8, 0xfffc, + 0xfffe, 0xffff] + cidr = 0 + for word in mask.split(':'): + if not word or int(word, 16) == 0: + break + cidr += bitCount.index(int(word, 16)) + + return cidr + + +def mask2cidr(mask): + if ':' in mask: + return ipv6mask2cidr(mask) + elif '.' in mask: + return ipv4mask2cidr(mask) + else: + return mask + + if __name__ == '__main__': import sys import random === modified file 'curtin/util.py' --- curtin/util.py 2016-10-03 18:00:41 +0000 +++ curtin/util.py 2016-10-03 18:55:20 +0000 @@ -16,18 +16,35 @@ # along with Curtin. If not, see . import argparse +import collections import errno import glob import json import os import platform +import re import shutil +import socket import subprocess import stat import sys import tempfile import time +# avoid the dependency to python3-six as used in cloud-init +try: + from urlparse import urlparse +except ImportError: + # python3 + # avoid triggering pylint, https://github.com/PyCQA/pylint/issues/769 + # pylint:disable=import-error,no-name-in-module + from urllib.parse import urlparse + +try: + string_types = (basestring,) +except NameError: + string_types = (str,) + from .log import LOG _INSTALLED_HELPERS_PATH = '/usr/lib/curtin/helpers' @@ -35,14 +52,22 @@ _LSB_RELEASE = {} +_DNS_REDIRECT_IP = None + +# matcher used in template rendering functions +BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)') + def _subp(args, data=None, rcs=None, env=None, capture=False, shell=False, - logstring=False, decode="replace"): + logstring=False, decode="replace", target=None): if rcs is None: rcs = [0] devnull_fp = None try: + if target_path(target) != "/": + args = ['chroot', target] + list(args) + if not logstring: LOG.debug(("Running command %s with allowed return codes %s" " (shell=%s, capture=%s)"), args, rcs, shell, capture) @@ -118,6 +143,8 @@ a list of times to sleep in between retries. After each failure subp will sleep for N seconds and then try again. A value of [1, 3] means to run, sleep 1, run, sleep 3, run and then return exit code. + :param target: + run the command as 'chroot target ' """ retries = [] if "retries" in kwargs: @@ -277,15 +304,29 @@ def write_file(filename, content, mode=0o644, omode="w"): + """ + write 'content' to file at 'filename' using python open mode 'omode'. + if mode is not set, then chmod file to mode. mode is 644 by default + """ ensure_dir(os.path.dirname(filename)) with open(filename, omode) as fp: fp.write(content) - os.chmod(filename, mode) - - -def load_file(path, mode="r"): + if mode: + os.chmod(filename, mode) + + +def load_file(path, mode="r", read_len=None, offset=0): with open(path, mode) as fp: - return fp.read() + if offset: + fp.seek(offset) + return fp.read(read_len) if read_len else fp.read() + + +def file_size(path): + """get the size of a file""" + with open(path, 'rb') as fp: + fp.seek(0, 2) + return fp.tell() def del_file(path): @@ -311,7 +352,7 @@ 'done', '']) - fpath = os.path.join(target, "usr/sbin/policy-rc.d") + fpath = target_path(target, "/usr/sbin/policy-rc.d") if os.path.isfile(fpath): return False @@ -322,7 +363,7 @@ def undisable_daemons_in_root(target): try: - os.unlink(os.path.join(target, "usr/sbin/policy-rc.d")) + os.unlink(target_path(target, "/usr/sbin/policy-rc.d")) except OSError as e: if e.errno != errno.ENOENT: raise @@ -334,7 +375,7 @@ def __init__(self, target, allow_daemons=False, sys_resolvconf=True): if target is None: target = "/" - self.target = os.path.abspath(target) + self.target = target_path(target) self.mounts = ["/dev", "/proc", "/sys"] self.umounts = [] self.disabled_daemons = False @@ -344,20 +385,21 @@ def __enter__(self): for p in self.mounts: - tpath = os.path.join(self.target, p[1:]) + tpath = target_path(self.target, p) if do_mount(p, tpath, opts='--bind'): self.umounts.append(tpath) if not self.allow_daemons: self.disabled_daemons = disable_daemons_in_root(self.target) - target_etc = os.path.join(self.target, "etc") + rconf = target_path(self.target, "/etc/resolv.conf") + target_etc = os.path.dirname(rconf) if self.target != "/" and os.path.isdir(target_etc): # never muck with resolv.conf on / rconf = os.path.join(target_etc, "resolv.conf") rtd = None try: - rtd = tempfile.mkdtemp(dir=os.path.dirname(rconf)) + rtd = tempfile.mkdtemp(dir=target_etc) tmp = os.path.join(rtd, "resolv.conf") os.rename(rconf, tmp) self.rconf_d = rtd @@ -375,25 +417,23 @@ undisable_daemons_in_root(self.target) # if /dev is to be unmounted, udevadm settle (LP: #1462139) - if os.path.join(self.target, "dev") in self.umounts: + if target_path(self.target, "/dev") in self.umounts: subp(['udevadm', 'settle']) for p in reversed(self.umounts): do_umount(p) - rconf = os.path.join(self.target, "etc", "resolv.conf") + rconf = target_path(self.target, "/etc/resolv.conf") if self.sys_resolvconf and self.rconf_d: os.rename(os.path.join(self.rconf_d, "resolv.conf"), rconf) shutil.rmtree(self.rconf_d) + def subp(self, *args, **kwargs): + kwargs['target'] = self.target + return subp(*args, **kwargs) -class RunInChroot(ChrootableTarget): - def __call__(self, args, **kwargs): - if self.target != "/": - chroot = ["chroot", self.target] - else: - chroot = [] - return subp(chroot + args, **kwargs) + def path(self, path): + return target_path(self.target, path) def is_exe(fpath): @@ -402,14 +442,13 @@ def which(program, search=None, target=None): - if target is None or os.path.realpath(target) == "/": - target = "/" + target = target_path(target) if os.path.sep in program: # if program had a '/' in it, then do not search PATH # 'which' does consider cwd here. (cd / && which bin/ls) = bin/ls # so effectively we set cwd to / (or target) - if is_exe(os.path.sep.join((target, program,))): + if is_exe(target_path(target, program)): return program if search is None: @@ -424,8 +463,9 @@ search = [os.path.abspath(p) for p in search] for path in search: - if is_exe(os.path.sep.join((target, path, program,))): - return os.path.sep.join((path, program,)) + ppath = os.path.sep.join((path, program)) + if is_exe(target_path(target, ppath)): + return ppath return None @@ -467,33 +507,39 @@ def get_architecture(target=None): - chroot = [] - if target is not None: - chroot = ['chroot', target] - out, _ = subp(chroot + ['dpkg', '--print-architecture'], - capture=True) + out, _ = subp(['dpkg', '--print-architecture'], capture=True, + target=target) return out.strip() def has_pkg_available(pkg, target=None): - chroot = [] - if target is not None: - chroot = ['chroot', target] - out, _ = subp(chroot + ['apt-cache', 'pkgnames'], capture=True) + out, _ = subp(['apt-cache', 'pkgnames'], capture=True, target=target) for item in out.splitlines(): if pkg == item.strip(): return True return False +def get_installed_packages(target=None): + (out, _) = subp(['dpkg-query', '--list'], target=target, capture=True) + + pkgs_inst = set() + for line in out.splitlines(): + try: + (state, pkg, other) = line.split(None, 2) + except ValueError: + continue + if state.startswith("hi") or state.startswith("ii"): + pkgs_inst.add(re.sub(":.*", "", pkg)) + + return pkgs_inst + + def has_pkg_installed(pkg, target=None): - chroot = [] - if target is not None: - chroot = ['chroot', target] try: - out, _ = subp(chroot + ['dpkg-query', '--show', '--showformat', - '${db:Status-Abbrev}', pkg], - capture=True) + out, _ = subp(['dpkg-query', '--show', '--showformat', + '${db:Status-Abbrev}', pkg], + capture=True, target=target) return out.rstrip() == "ii" except ProcessExecutionError: return False @@ -542,13 +588,9 @@ """Use dpkg-query to extract package pkg's version string and parse the version string into a dictionary """ - chroot = [] - if target is not None: - chroot = ['chroot', target] try: - out, _ = subp(chroot + ['dpkg-query', '--show', '--showformat', - '${Version}', pkg], - capture=True) + out, _ = subp(['dpkg-query', '--show', '--showformat', + '${Version}', pkg], capture=True, target=target) raw = out.rstrip() return parse_dpkg_version(raw, name=pkg, semx=semx) except ProcessExecutionError: @@ -600,11 +642,11 @@ if comment.endswith("\n"): comment = comment[:-1] - marker = os.path.join(target, marker) + marker = target_path(target, marker) # if marker exists, check if there are files that would make it obsolete - listfiles = [os.path.join(target, "etc/apt/sources.list")] + listfiles = [target_path(target, "/etc/apt/sources.list")] listfiles += glob.glob( - os.path.join(target, "etc/apt/sources.list.d/*.list")) + target_path(target, "etc/apt/sources.list.d/*.list")) if os.path.exists(marker) and not force: if len(find_newer(marker, listfiles)) == 0: @@ -612,7 +654,7 @@ restore_perms = [] - abs_tmpdir = tempfile.mkdtemp(dir=os.path.join(target, 'tmp')) + abs_tmpdir = tempfile.mkdtemp(dir=target_path(target, "/tmp")) try: abs_slist = abs_tmpdir + "/sources.list" abs_slistd = abs_tmpdir + "/sources.list.d" @@ -621,8 +663,8 @@ ch_slistd = ch_tmpdir + "/sources.list.d" # this file gets executed on apt-get update sometimes. (LP: #1527710) - motd_update = os.path.join( - target, "usr/lib/update-notifier/update-motd-updates-available") + motd_update = target_path( + target, "/usr/lib/update-notifier/update-motd-updates-available") pmode = set_unexecutable(motd_update) if pmode is not None: restore_perms.append((motd_update, pmode),) @@ -647,8 +689,8 @@ 'update'] # do not using 'run_apt_command' so we can use 'retries' to subp - with RunInChroot(target, allow_daemons=True) as inchroot: - inchroot(update_cmd, env=env, retries=retries) + with ChrootableTarget(target, allow_daemons=True) as inchroot: + inchroot.subp(update_cmd, env=env, retries=retries) finally: for fname, perms in restore_perms: os.chmod(fname, perms) @@ -685,9 +727,8 @@ return env, cmd apt_update(target, env=env, comment=' '.join(cmd)) - ric = RunInChroot(target, allow_daemons=allow_daemons) - with ric as inchroot: - return inchroot(cmd, env=env) + with ChrootableTarget(target, allow_daemons=allow_daemons) as inchroot: + return inchroot.subp(cmd, env=env) def system_upgrade(aptopts=None, target=None, env=None, allow_daemons=False): @@ -716,7 +757,7 @@ """ Look for "hook" in "target" and run it """ - target_hook = os.path.join(target, 'curtin', hook) + target_hook = target_path(target, '/curtin/' + hook) if os.path.isfile(target_hook): LOG.debug("running %s" % target_hook) subp([target_hook]) @@ -828,6 +869,18 @@ return val +def bytes2human(size): + """convert size in bytes to human readable""" + if not (isinstance(size, (int, float)) and + int(size) == size and + int(size) >= 0): + raise ValueError('size must be a integral value') + mpliers = {'B': 1, 'K': 2 ** 10, 'M': 2 ** 20, 'G': 2 ** 30, 'T': 2 ** 40} + unit_order = sorted(mpliers, key=lambda x: -1 * mpliers[x]) + unit = next((u for u in unit_order if (size / mpliers[u]) >= 1), 'B') + return str(int(size / mpliers[unit])) + unit + + def import_module(import_str): """Import a module.""" __import__(import_str) @@ -843,30 +896,42 @@ def is_file_not_found_exc(exc): - return (isinstance(exc, IOError) and exc.errno == errno.ENOENT) - - -def lsb_release(): + return (isinstance(exc, (IOError, OSError)) and + hasattr(exc, 'errno') and + exc.errno in (errno.ENOENT, errno.EIO, errno.ENXIO)) + + +def _lsb_release(target=None): fmap = {'Codename': 'codename', 'Description': 'description', 'Distributor ID': 'id', 'Release': 'release'} + + data = {} + try: + out, _ = subp(['lsb_release', '--all'], capture=True, target=target) + for line in out.splitlines(): + fname, _, val = line.partition(":") + if fname in fmap: + data[fmap[fname]] = val.strip() + missing = [k for k in fmap.values() if k not in data] + if len(missing): + LOG.warn("Missing fields in lsb_release --all output: %s", + ','.join(missing)) + + except ProcessExecutionError as err: + LOG.warn("Unable to get lsb_release --all: %s", err) + data = {v: "UNAVAILABLE" for v in fmap.values()} + + return data + + +def lsb_release(target=None): + if target_path(target) != "/": + # do not use or update cache if target is provided + return _lsb_release(target) + global _LSB_RELEASE if not _LSB_RELEASE: - data = {} - try: - out, err = subp(['lsb_release', '--all'], capture=True) - for line in out.splitlines(): - fname, tok, val = line.partition(":") - if fname in fmap: - data[fmap[fname]] = val.strip() - missing = [k for k in fmap.values() if k not in data] - if len(missing): - LOG.warn("Missing fields in lsb_release --all output: %s", - ','.join(missing)) - - except ProcessExecutionError as e: - LOG.warn("Unable to get lsb_release --all: %s", e) - data = {v: "UNAVAILABLE" for v in fmap.values()} - + data = _lsb_release() _LSB_RELEASE.update(data) return _LSB_RELEASE @@ -881,8 +946,7 @@ def json_dumps(data): - return json.dumps(data, indent=1, sort_keys=True, - separators=(',', ': ')).encode('utf-8') + return json.dumps(data, indent=1, sort_keys=True, separators=(',', ': ')) def get_platform_arch(): @@ -895,4 +959,137 @@ } return platform2arch.get(platform.machine(), platform.machine()) + +def basic_template_render(content, params): + """This does simple replacement of bash variable like templates. + + It identifies patterns like ${a} or $a and can also identify patterns like + ${a.b} or $a.b which will look for a key 'b' in the dictionary rooted + by key 'a'. + """ + + def replacer(match): + """ replacer + replacer used in regex match to replace content + """ + # Only 1 of the 2 groups will actually have a valid entry. + name = match.group(1) + if name is None: + name = match.group(2) + if name is None: + raise RuntimeError("Match encountered but no valid group present") + path = collections.deque(name.split(".")) + selected_params = params + while len(path) > 1: + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not traverse into" + " non-dictionary '%s' of type %s while" + " looking for subkey '%s'" + % (selected_params, + selected_params.__class__.__name__, + key)) + selected_params = selected_params[key] + key = path.popleft() + if not isinstance(selected_params, dict): + raise TypeError("Can not extract key '%s' from non-dictionary" + " '%s' of type %s" + % (key, selected_params, + selected_params.__class__.__name__)) + return str(selected_params[key]) + + return BASIC_MATCHER.sub(replacer, content) + + +def render_string(content, params): + """ render_string + render a string following replacement rules as defined in + basic_template_render returning the string + """ + if not params: + params = {} + return basic_template_render(content, params) + + +def is_resolvable(name): + """determine if a url is resolvable, return a boolean + This also attempts to be resilent against dns redirection. + + Note, that normal nsswitch resolution is used here. So in order + to avoid any utilization of 'search' entries in /etc/resolv.conf + we have to append '.'. + + The top level 'invalid' domain is invalid per RFC. And example.com + should also not exist. The random entry will be resolved inside + the search list. + """ + global _DNS_REDIRECT_IP + if _DNS_REDIRECT_IP is None: + badips = set() + badnames = ("does-not-exist.example.com.", "example.invalid.") + badresults = {} + for iname in badnames: + try: + result = socket.getaddrinfo(iname, None, 0, 0, + socket.SOCK_STREAM, + socket.AI_CANONNAME) + badresults[iname] = [] + for (_, _, _, cname, sockaddr) in result: + badresults[iname].append("%s: %s" % (cname, sockaddr[0])) + badips.add(sockaddr[0]) + except (socket.gaierror, socket.error): + pass + _DNS_REDIRECT_IP = badips + if badresults: + LOG.debug("detected dns redirection: %s", badresults) + + try: + result = socket.getaddrinfo(name, None) + # check first result's sockaddr field + addr = result[0][4][0] + if addr in _DNS_REDIRECT_IP: + LOG.debug("dns %s in _DNS_REDIRECT_IP", name) + return False + LOG.debug("dns %s resolved to '%s'", name, result) + return True + except (socket.gaierror, socket.error): + LOG.debug("dns %s failed to resolve", name) + return False + + +def is_resolvable_url(url): + """determine if this url is resolvable (existing or ip).""" + return is_resolvable(urlparse(url).hostname) + + +def target_path(target, path=None): + # return 'path' inside target, accepting target as None + if target in (None, ""): + target = "/" + elif not isinstance(target, string_types): + raise ValueError("Unexpected input for target: %s" % target) + else: + target = os.path.abspath(target) + # abspath("//") returns "//" specifically for 2 slashes. + if target.startswith("//"): + target = target[1:] + + if not path: + return target + + # os.path.join("/etc", "/foo") returns "/foo". Chomp all leading /. + while len(path) and path[0] == "/": + path = path[1:] + + return os.path.join(target, path) + + +class RunInChroot(ChrootableTarget): + """Backwards compatibility for RunInChroot (LP: #1617375). + It needs to work like: + with RunInChroot("/target") as in_chroot: + in_chroot(["your", "chrooted", "command"])""" + __call__ = ChrootableTarget.subp + + # vi: ts=4 expandtab syntax=python === modified file 'debian/changelog' --- debian/changelog 2016-10-03 17:23:32 +0000 +++ debian/changelog 2016-10-03 18:55:20 +0000 @@ -1,8 +1,38 @@ -curtin (0.1.0~bzr399-0ubuntu1~16.04.1ubuntu1) UNRELEASED; urgency=medium +curtin (0.1.0~bzr425-0ubuntu1~16.04.1) xenial-proposed; urgency=medium + [ Scott Moser ] * debian/new-upstream-snapshot: add writing of debian changelog entries. - -- Scott Moser Mon, 03 Oct 2016 13:23:11 -0400 + [ Ryan Harper ] + * New upstream snapshot. + - unittest,tox.ini: catch and fix issue with trusty-level mock of open + - block/mdadm: add option to ignore mdadm_assemble errors (LP: #1618429) + - curtin/doc: overhaul curtin documentation for readthedocs.org (LP: #1351085) + - curtin.util: re-add support for RunInChroot (LP: #1617375) + - curtin/net: overhaul of eni rendering to handle mixed ipv4/ipv6 configs + - curtin.block: refactor clear_holders logic into block.clear_holders and cli cmd + - curtin.apply_net should exit non-zero upon exception. (LP: #1615780) + - apt: fix bug in disable_suites if sources.list line is blank. + - vmtests: disable Wily in vmtests + - Fix the unittests for test_apt_source. + - get CURTIN_VMTEST_PARALLEL shown correctly in jenkins-runner output + - fix vmtest check_file_strippedline to strip lines before comparing + - fix whitespace damage in tests/vmtests/__init__.py + - fix dpkg-reconfigure when debconf_selections was provided. (LP: #1609614) + - fix apt tests on non-intel arch + - Add apt features to curtin. (LP: #1574113) + - vmtest: easier use of parallel and controlling timeouts + - mkfs.vfat: add force flag for formating whole disks (LP: #1597923) + - block.mkfs: fix sectorsize flag (LP: #1597522) + - block_meta: cleanup use of sys_block_path and handle cciss knames (LP: #1562249) + - block.get_blockdev_sector_size: handle _lsblock multi result return (LP: #1598310) + - util: add target (chroot) support to subp, add target_path helper. + - block_meta: fallback to parted if blkid does not produce output (LP: #1524031) + - commands.block_wipe: correct default wipe mode to 'superblock' + - tox.ini: run coverage normally rather than separately + - move uefi boot knowledge from launch and vmtest to xkvm + + -- Ryan Harper Mon, 03 Oct 2016 13:43:54 -0500 curtin (0.1.0~bzr399-0ubuntu1~16.04.1) xenial-proposed; urgency=medium === modified file 'doc/conf.py' --- doc/conf.py 2015-10-02 16:19:07 +0000 +++ doc/conf.py 2016-10-03 18:55:20 +0000 @@ -13,6 +13,11 @@ import sys, os +# Fix path so we can import curtin.__version__ +sys.path.insert(1, os.path.realpath(os.path.join( + os.path.dirname(__file__), '..'))) +import curtin + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -41,16 +46,16 @@ # General information about the project. project = u'curtin' -copyright = u'2013, Scott Moser' +copyright = u'2016, Scott Moser, Ryan Harper' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.3' +version = curtin.__version__ # The full version, including alpha/beta/rc tags. -release = '0.3' +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -93,6 +98,18 @@ # a list of builtin themes. html_theme = 'classic' +# on_rtd is whether we are on readthedocs.org, this line of code grabbed from +# docs.readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# otherwise, readthedocs.org uses their theme by default, so no need to specify +# it + # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. @@ -120,7 +137,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['static'] +#html_static_path = ['static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. === removed file 'doc/devel/README-vmtest.txt' --- doc/devel/README-vmtest.txt 2016-02-12 21:54:46 +0000 +++ doc/devel/README-vmtest.txt 1970-01-01 00:00:00 +0000 @@ -1,152 +0,0 @@ -== Background == -Curtin includes a mechanism called 'vmtest' that allows it to actually -do installs and validate a number of configurations. - -The general flow of the vmtests is: - 1. each test has an associated yaml config file for curtin in examples/tests - 2. uses curtin-pack to create the user-data for cloud-init to trigger install - 3. create and install a system using 'tools/launch'. - 3.1 The install environment is booted from a maas ephemeral image. - 3.2 kernel & initrd used are from maas images (not part of the image) - 3.3 network by default is handled via user networking - 3.4 It creates all empty disks required - 3.5 cloud-init datasource is provided by launch - a) like: ds=nocloud-net;seedfrom=http://10.7.0.41:41518/ - provided by python webserver start_http - b) via -drive file=/tmp/launch.8VOiOn/seed.img,if=virtio,media=cdrom - as a seed disk (if booted without external kernel) - 3.6 dependencies and other preparations are installed at the beginning by - curtin inside the ephemeral image prior to configuring the target - 4. power off the system. - 5. configure a 'NoCloud' datasource seed image that provides scripts that - will run on first boot. - 5.1 this will contain all our code to gather health data on the install - 5.2 by cloud-init design this runs only once per instance, if you start - the system again this won't be called again - 6. boot the installed system with 'tools/xkvm'. - 6.1 reuses the disks that were installed/configured in the former steps - 6.2 also adds an output disk - 6.3 additionally the seed image for the data gathering is added - 6.4 On this boot it will run the provided scripts, write their output to a - "data" disk and then shut itself down. - 7. extract the data from the output disk - 8. vmtest python code now verifies if the output is as expected. - -== Debugging == -At 3.1 - - one can pull data out of the maas image with - sudo mount-image-callback your.img -- sh -c 'COMMAND' - e.g. sudo mount-image-callback your.img -- sh -c 'cp $MOUNTPOINT/boot/* .' -At step 3.6 -> 4. - - tools/launch can be called in a way to give you console access - to do so just call tools/launch but drop the -serial=x parameter. - One might want to change "'power_state': {'mode': 'poweroff'}" to avoid - the auto reboot before getting control - Replace the directory usually seen in the launch calls with a clean fresh - directory - - In /curtin curtin and its config can be found - - if the system gets that far cloud-init will create a user ubuntu/passw0rd - - otherwise one can use a cloud-image from https://cloud-images.ubuntu.com/ - and add a backdoor user via - bzr branch lp:~maas-maintainers/maas/backdoor-image backdoor-image - sudo ./backdoor-image -v --user= --password-auth --password= IMG -At step 6 -> 7 - - You might want to keep all the temporary images around. - To do so you can set CURTIN_VMTEST_KEEP_DATA_PASS=all: - export CURTIN_VMTEST_KEEP_DATA_PASS=all CURTIN_VMTEST_KEEP_DATA_FAIL=all - That will keep the /tmp/tmpXXXXX directories and all files in there for - further execution. -At step 7 - - You might want to take a look at the output disk yourself. - It is a normal qcow image, so one can use mount-image-callback as described - above - - to invoke xkvm on your own take the command you see in the output and - remove the "-serial ..." but add -nographic instead - For graphical console one can add --vnc 127.0.0.1:1 - -== Setup == -In order to run vmtest you'll need some dependencies. To get them, you -can run: - make vmtest-deps - -That will install all necessary dependencies. - -== Running == -Running tests is done most simply by: - - make vmtest - -If you wish to all tests in test_network.py, do so with: - sudo PATH=$PWD/tools:$PATH nosetests3 tests/vmtests/test_network.py - -Or run a single test with: - sudo PATH=$PWD/tools:$PATH nosetests3 tests/vmtests/test_network.py:WilyTestBasic - -Note: - * currently, the tests have to run as root. The reason for this is that - the kernel and initramfs to boot are extracted from the maas ephemeral - image. This should be fixed at some point, and then 'make vmtest' - - The tests themselves don't actually have to run as root, but the - test setup does. - * the 'tools' directory must be in your path. - * test will set apt_proxy in the guests to the value of - 'apt_proxy' environment variable. If that is not set it will - look at the host's apt config and read 'Acquire::HTTP::Proxy' - -== Environment Variables == -Some environment variables affect the running of vmtest - * apt_proxy: - test will set apt_proxy in the guests to the value of 'apt_proxy'. - If that is not set it will look at the host's apt config and read - 'Acquire::HTTP::Proxy' - - * CURTIN_VMTEST_KEEP_DATA_PASS CURTIN_VMTEST_KEEP_DATA_FAIL: - default: - CURTIN_VMTEST_KEEP_DATA_PASS=none - CURTIN_VMTEST_KEEP_DATA_FAIL=all - These 2 variables determine what portions of the temporary - test data are kept. - - The variables contain a comma ',' delimited list of directories - that should be kept in the case of pass or fail. Additionally, - the values 'all' and 'none' are accepted. - - Each vmtest that runs has its own sub-directory under the top level - CURTIN_VMTEST_TOPDIR. In that directory are directories: - boot: inputs to the system boot (after install) - install: install phase related files - disks: the disks used for installation and boot - logs: install and boot logs - collect: data collected by the boot phase - - * CURTIN_VMTEST_TOPDIR: default $TMPDIR/vmtest- - vmtest puts all test data under this value. By default, it creates - a directory in TMPDIR (/tmp) named with as "vmtest-" - - If you set this value, you must ensure that the directory is either - non-existant or clean. - - * CURTIN_VMTEST_LOG: default $TMPDIR/vmtest-.log - vmtest writes extended log information to this file. - The default puts the log along side the TOPDIR. - - * CURTIN_VMTEST_IMAGE_SYNC: default false (boolean) - if set to true, each run will attempt a sync of images. - If you want to make sure images are always up to date, then set to true. - - * CURTIN_VMTEST_BRIDGE: default 'user' - the network devices will be attached to this bridge. The default is - 'user', which means to use qemu user mode networking. Set it to - 'virbr0' or 'lxcbr0' to use those bridges and then be able to ssh - in directly. - - * IMAGE_DIR: default /srv/images - vmtest keeps a mirror of maas ephemeral images in this directory. - - * IMAGES_TO_KEEP: default 1 - keep this number of images of each release in the IMAGE_DIR. - -Environment 'boolean' values: - For boolean environment variables the value is considered True - if it is any value other than case insensitive 'false', '' or "0" === removed file 'doc/devel/README.txt' --- doc/devel/README.txt 2015-03-11 13:19:43 +0000 +++ doc/devel/README.txt 1970-01-01 00:00:00 +0000 @@ -1,55 +0,0 @@ -## curtin development ## - -This document describes how to use kvm and ubuntu cloud images -to develop curtin or test install configurations inside kvm. - -## get some dependencies ## -sudo apt-get -qy install kvm libvirt-bin cloud-utils bzr - -## get cloud image to boot (-disk1.img) and one to install (-root.tar.gz) -mkdir -p ~/download -DLDIR=$( cd ~/download && pwd ) -rel="trusty" -arch=amd64 -burl="http://cloud-images.ubuntu.com/$rel/current/" -for f in $rel-server-cloudimg-${arch}-root.tar.gz $rel-server-cloudimg-${arch}-disk1.img; do - wget "$burl/$f" -O $DLDIR/$f; done -( cd $DLDIR && qemu-img convert -O qcow $rel-server-cloudimg-${arch}-disk1.img $rel-server-cloudimg-${arch}-disk1.qcow2) - -BOOTIMG="$DLDIR/$rel-server-cloudimg-${arch}-disk1.qcow2" -ROOTTGZ="$DLDIR/$rel-server-cloudimg-${arch}-root.tar.gz" - -## get curtin -mkdir -p ~/src -bzr init-repo ~/src/curtin -( cd ~/src/curtin && bzr branch lp:curtin trunk.dist ) -( cd ~/src/curtin && bzr branch trunk.dist trunk ) - -## work with curtin -cd ~/src/curtin/trunk -# use 'launch' to launch a kvm instance with user data to pack -# up local curtin and run it inside instance. -./tools/launch $BOOTIMG --publish $ROOTTGZ -- curtin install "PUBURL/${ROOTTGZ##*/}" - -## notes about 'launch' ## - * launch has --help so you can see that for some info. - * '--publish' adds a web server at ${HTTP_PORT:-9923} - and puts the files you want available there. You can reference - this url in config or cmdline with 'PUBURL'. For example - '--publish foo.img' will put 'foo.img' at PUBURL/foo.img. - * launch sets 'ubuntu' user password to 'passw0rd' - * launch runs 'kvm -curses' - kvm -curses keyboard info: - 'alt-2' to go to qemu console - * launch puts serial console to 'serial.log' (look there for stuff) - * when logged in - * you can look at /var/log/cloud-init-output.log - * archive should be extracted in /curtin - * shell archive should be in /var/lib/cloud/instance/scripts/part-002 - * when logged in, and archive available at - - -## other notes ## - * need to add '--install-deps' or something for curtin - cloud-image in 12.04 has no 'python3' - ideally 'curtin --install-deps install' would get the things it needs === added file 'doc/devel/clear_holders_doc.txt' --- doc/devel/clear_holders_doc.txt 1970-01-01 00:00:00 +0000 +++ doc/devel/clear_holders_doc.txt 2016-10-03 18:55:20 +0000 @@ -0,0 +1,85 @@ +The new version of clear_holders is based around a data structure called a +holder_tree which represents the current storage hirearchy above a specified +starting device. Each node in a holders tree contains data about the node and a +key 'holders' which contains a list of all nodes that depend on it. The keys in +a holdres_tree node are: + - device: the path to the device in /sys/class/block + - dev_type: what type of storage layer the device is. possible values: + - disk + - lvm + - crypt + - raid + - bcache + - disk + - name: the kname of the device (used for display) + - holders: holders_trees for devices depending on the current device + +A holders tree can be generated for a device using the function +clear_holders.gen_holders_tree. The device can be specified either as a path in +/sys/class/block or as a path in /dev. + +The new implementation of block.clear_holders shuts down storage devices in a +holders tree starting from the leaves of the tree and ascending towards the +root. The old implementation of clear_holders ascended up each path of the tree +separately, in a pattern similar to depth first search. The problem with the +old implementation is that in some cases either an attempt would be made to +remove one storage device while other devices depended on it or clear_holders +would attempt to shut down the same storage device several times. In order to +cope with this the old version of clear_holders had logic to handle expected +failures and hope for the best moving forward. The new version of clear_holders +is able to run without many anticipated failures. + +The logic to plan what order to shut down storage layers in is in +clear_holders.plan_shutdown_holders_trees. This function accepts either a +single holders tree or a list of holders trees. When run with a list of holders +trees, it assumes that all of these trees start at basically the same layer in +the overall storage hirearcy for the system (i.e. a list of holders trees +starting from all of the target installation disks). This function returns a +list of dictionaries, with each dictionary containing the keys: + - device: the path to the device in /sys/class/block + - dev_type: what type of storage layer the device is. possible values: + - disk + - lvm + - crypt + - raid + - bcache + - disk + - level: the level of the device in the current storage hirearchy + (starting from 0) + +The items in the list returned by clear_holders.plan_shutdown_holders_trees +should be processed in order to make sure the holders trees are shutdown fully + +The main interface for clear_holders is the function +clear_holders.clear_holders. If the system has just been booted it could be +beneficial to run the function clear_holders.start_clear_holders_deps before +using clear_holders.clear_holders. This ensures clear_holders will be able to +properly storage devices. The function clear_holders.clear_holders can be +passed either a single device or a list of devices and will shut down all +storage devices above the device(s). The devices can be specified either by +path in /dev or by path in /sys/class/block. + +In order to test if a device or devices are free to be partitioned/formatted, +the function clear_holders.assert_clear can be passed either a single device or +a list of devices, with devices specified either by path in /dev or by path in +/sys/class/block. If there are any storage devices that depend on one of the +devices passed to clear_holders.assert_clear, then an OSError will be raised. +If clear_holders.assert_clear does not raise any errors, then the devices +specified should be ready for partitioning. + +It is possible to query further information about storage devices using +clear_holders. + +Holders for a individual device can be queried using clear_holders.get_holders. +Results are returned as a list or knames for holding devices. + +A holders tree can be printed in a human readable format using +clear_holders.format_holders_tree(). Example output: +sda +|-- sda1 +|-- sda2 +`-- sda5 + `-- dm-0 + |-- dm-1 + `-- dm-2 + `-- dm-3 === modified file 'doc/index.rst' --- doc/index.rst 2015-10-02 16:19:07 +0000 +++ doc/index.rst 2016-10-03 18:55:20 +0000 @@ -13,7 +13,13 @@ :maxdepth: 2 topics/overview + topics/config + topics/apt_source + topics/networking + topics/storage topics/reporting + topics/development + topics/integration-testing === added file 'doc/topics/apt_source.rst' --- doc/topics/apt_source.rst 1970-01-01 00:00:00 +0000 +++ doc/topics/apt_source.rst 2016-10-03 18:55:20 +0000 @@ -0,0 +1,164 @@ +========== +APT Source +========== + +This part of curtin is meant to allow influencing the apt behaviour and configuration. + +By default - if no apt config is provided - it does nothing. That keeps behavior compatible on upgrades. + +The feature has an optional target argument which - by default - is used to modify the environment that curtin currently installs (@TARGET_MOUNT_POINT). + +Features +~~~~~~~~ + +* Add PGP keys to the APT trusted keyring + + - add via short keyid + + - add via long key fingerprint + + - specify a custom keyserver to pull from + + - add raw keys (which makes you independent of keyservers) + +* Influence global apt configuration + + - adding ppa's + + - replacing mirror, security mirror and release in sources.list + + - able to provide a fully custom template for sources.list + + - add arbitrary apt.conf settings + + - provide debconf configurations + + - disabling suites (=pockets) + + - per architecture mirror definition + + +Configuration +~~~~~~~~~~~~~ + +The general configuration of the apt feature is under an element called ``apt``. + +This can have various "global" subelements as listed in the examples below. +The file ``apt-source.yaml`` holds more examples. + +These global configurations are valid throughput all of the apt feature. +So for exmaple a global specification of a ``primary`` mirror will apply to all rendered sources entries. + +Then there is a section ``sources`` which can hold any number of source subelements itself. +The key is the filename and will be prepended by /etc/apt/sources.list.d/ if it doesn't start with a ``/``. +There are certain cases - where no content is written into a source.list file where the filename will be ignored - yet it can still be used as index for merging. + +The values inside the entries consist of the following optional entries + +* ``source``: a sources.list entry (some variable replacements apply) + +* ``keyid``: providing a key to import via shortid or fingerprint + +* ``key``: providing a raw PGP key + +* ``keyserver``: specify an alternate keyserver to pull keys from that were specified by keyid + +The section "sources" is is a dictionary (unlike most block/net configs which are lists). This format allows merging between multiple input files than a list like :: + + sources: + s1: {'key': 'key1', 'source': 'source1'} + + sources: + s2: {'key': 'key2'} + s1: {'keyserver': 'foo'} + + This would be merged into + s1: {'key': 'key1', 'source': 'source1', keyserver: 'foo'} + s2: {'key': 'key2'} + +Here is just one of the most common examples for this feature: install with curtin in an isolated environment (derived repository): + +For that we need to: +* insert the PGP key of the local repository to be trusted + + - since you are locked down you can't pull from keyserver.ubuntu.com + + - if you have an internal keyserver you could pull from there, but let us assume you don't even have that; so you have to provide the raw key + + - in the example I'll use the key of the "Ubuntu CD Image Automatic Signing Key" which makes no sense as it is in the trusted keyring anyway, but it is a good example. (Also the key is shortened to stay readable) + +:: + + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + mQGiBEFEnz8RBAC7LstGsKD7McXZgd58oN68KquARLBl6rjA2vdhwl77KkPPOr3O + RwIbDAAKCRBAl26vQ30FtdxYAJsFjU+xbex7gevyGQ2/mhqidES4MwCggqQyo+w1 + Twx6DKLF+3rF5nf1F3Q= + =PBAe + -----END PGP PUBLIC KEY BLOCK----- + +* replace the mirrors used to some mirrors available inside the isolated environment for apt to pull repository data from. + + - lets consider we have a local mirror at ``mymirror.local`` but otherwise following the usual paths + + - make an example with a partial mirror that doesn't mirror the backports suite, so backports have to be disabled + +That would be specified as :: + + apt: + primary: + - arches [default] + uri: http://mymirror.local/ubuntu/ + disable_suites: [backports] + sources: + localrepokey: + key: | # full key as block + -----BEGIN PGP PUBLIC KEY BLOCK----- + Version: GnuPG v1 + + mQGiBEFEnz8RBAC7LstGsKD7McXZgd58oN68KquARLBl6rjA2vdhwl77KkPPOr3O + RwIbDAAKCRBAl26vQ30FtdxYAJsFjU+xbex7gevyGQ2/mhqidES4MwCggqQyo+w1 + Twx6DKLF+3rF5nf1F3Q= + =PBAe + -----END PGP PUBLIC KEY BLOCK----- + +The file examples/apt-source.yaml holds various further examples that can be configured with this feature. + + +Common snippets +~~~~~~~~~~~~~~~ +This is a collection of additional ideas people can use the feature for customizing their to-be-installed system. + +* enable proposed on installing + +:: + + apt: + sources: + proposed.list: deb $MIRROR $RELEASE-proposed main restricted universe multiverse + +* Make debug symbols available + +:: + + apt: + sources: + ddebs.list: | + deb http://ddebs.ubuntu.com $RELEASE main restricted universe multiverse +  deb http://ddebs.ubuntu.com $RELEASE-updates main restricted universe multiverse +  deb http://ddebs.ubuntu.com $RELEASE-security main restricted universe multiverse + deb http://ddebs.ubuntu.com $RELEASE-proposed main restricted universe multiverse + +Timing +~~~~~~ +The feature is implemented at the stage of curthooks_commands, which runs just after curtin has extracted the image to the target. +Additionally it can be ran as standalong command "curtin -v --config apt-config". + +This will pick up the target from the environment variable that is set by curtin, if you want to use it to a different target or outside of usual curtin handling you can add ``--target `` to it to overwrite the target path. +This target should have at least a minimal system with apt, apt-add-repository and dpkg being installed for the functionality to work. + + +Dependencies +~~~~~~~~~~~~ +Cloud-init might need to resolve dependencies and install packages in the ephemeral environment to run curtin. +Therefore it is recommended to not only provide an apt configuration to curtin for the target, but also one to the install environment via cloud-init. === added file 'doc/topics/config.rst' --- doc/topics/config.rst 1970-01-01 00:00:00 +0000 +++ doc/topics/config.rst 2016-10-03 18:55:20 +0000 @@ -0,0 +1,551 @@ +==================== +Curtin Configuration +==================== + +Curtin exposes a number of configuration options for controlling Curtin +behavior during installation. + + +Configuration options +--------------------- +Curtin's top level config keys are as follows: + + +- apt_mirrors (``apt_mirrors``) +- apt_proxy (``apt_proxy``) +- block-meta (``block``) +- debconf_selections (``debconf_selections``) +- disable_overlayroot (``disable_overlayroot``) +- grub (``grub``) +- http_proxy (``http_proxy``) +- install (``install``) +- kernel (``kernel``) +- kexec (``kexec``) +- multipath (``multipath``) +- network (``network``) +- power_state (``power_state``) +- reporting (``reporting``) +- restore_dist_interfaces: (``restore_dist_interfaces``) +- sources (``sources``) +- stages (``stages``) +- storage (``storage``) +- swap (``swap``) +- system_upgrade (``system_upgrade``) +- write_files (``write_files``) + + +apt_mirrors +~~~~~~~~~~~ +Configure APT mirrors for ``ubuntu_archive`` and ``ubuntu_security`` + +**ubuntu_archive**: ** + +**ubuntu_security**: ** + +If the target OS includes /etc/apt/sources.list, Curtin will replace +the default values for each key set with the supplied mirror URL. + +**Example**:: + + apt_mirrors: + ubuntu_archive: http://local.archive/ubuntu + ubuntu_security: http://local.archive/ubuntu + + +apt_proxy +~~~~~~~~~ +Curtin will configure an APT HTTP proxy in the target OS + +**apt_proxy**: ** + +**Example**:: + + apt_proxy: http://squid.mirror:3267/ + + +block-meta +~~~~~~~~~~ +Configure how Curtin selects and configures disks on the target +system without providing a custom configuration (mode=simple). + +**devices**: ** + +The ``devices`` parameter is a list of block device paths that Curtin may +select from with choosing where to install the OS. + +**boot-partition**: ** + +The ``boot-partition`` parameter controls how to configure the boot partition +with the following parameters: + +**enabled**: ** + +Enabled will forcibly setup a partition on the target device for booting. + +**format**: *<['uefi', 'gpt', 'prep', 'mbr']>* + +Specify the partition format. Some formats, like ``uefi`` and ``prep`` +are restricted by platform characteristics. + +**fstype**: ** + +Specify the filesystem format on the boot partition. + +**label**: ** + +Specify the filesystem label on the boot partition. + +**Example**:: + + block-meta: + devices: + - /dev/sda + - /dev/sdb + boot-partition: + - enabled: True + format: gpt + fstype: ext4 + label: my-boot-partition + + +debconf_selections +~~~~~~~~~~~~~~~~~~ +Curtin will update the target with debconf set-selection values. Users will +need to be familiar with the package debconf options. Users can probe a +packages' debconf settings by using ``debconf-get-selections``. + +**selection_name**: ** + +``debconf-set-selections`` is in the form:: + + + +**Example**:: + + debconf_selections: + set1: | + cloud-init cloud-init/datasources multiselect MAAS + lxd lxd/bridge-name string lxdbr0 + set2: lxd lxd/setup-bridge boolean true + + + +disable_overlayroot +~~~~~~~~~~~~~~~~~~~ +Curtin disables overlayroot in the target by default. + +**disable_overlayroot**: ** + +**Example**:: + + disable_overlayroot: False + + +grub +~~~~ +Curtin configures grub as the target machine's boot loader. Users +can control a few options to tailor how the system will boot after +installation. + +**install_devices**: ** + +Specify a list of devices onto which grub will attempt to install. + +**replace_linux_default**: ** + +Controls whether grub-install will update the Linux Default target +value during installation. + +**update_nvram**: ** + +Certain platforms, like ``uefi`` and ``prep`` systems utilize +NVRAM to hold boot configuration settings which control the order in +which devices are booted. Curtin by default will not attempt to +update the NVRAM settings to preserve the system configuration. +Users may want to force NVRAM to be updated such that the next boot +of the system will boot from the installed device. + +**Example**:: + + grub: + install_devices: + - /dev/sda1 + replace_linux_default: False + update_nvram: True + + +http_proxy +~~~~~~~~~~ +Curtin will export ``http_proxy`` value into the installer environment. + +**http_proxy**: ** + +**Example**:: + + http_proxy: http://squid.proxy:3728/ + + + +install +~~~~~~~ +Configure Curtin's install options. + +**log_file**: ** + +Curtin logs install progress by default to /var/log/curtin/install.log + +**post_files**: ** + +Curtin by default will post the ``log_file`` value to any configured reporter. + +**save_install_config**: ** + +Curtin will save the merged configuration data into the target OS at +the path of ``save_install_config``. This defaults to /root/curtin-install-cfg.yaml + +**Example**:: + + install: + log_file: /tmp/install.log + post_files: + - /tmp/install.log + - /var/log/syslog + save_install_config: /root/myconf.yaml + + +kernel +~~~~~~ +Configure how Curtin selects which kernel to install into the target image. +If ``kernel`` is not configured, Curtin will use the default mapping below +and determine which ``package`` value by looking up the current release +and current kernel version running. + + +**fallback-package**: ** + +Specify a kernel package name to be used if the default package is not +available. + +**mapping**: ** + +Default mapping for Releases to package names is as follows:: + + precise: + 3.2.0: + 3.5.0: -lts-quantal + 3.8.0: -lts-raring + 3.11.0: -lts-saucy + 3.13.0: -lts-trusty + trusty: + 3.13.0: + 3.16.0: -lts-utopic + 3.19.0: -lts-vivid + 4.2.0: -lts-wily + 4.4.0: -lts-xenial + xenial: + 4.3.0: + 4.4.0: + + +**package**: ** + +Specify the exact package to install in the target OS. + +**Example**:: + + kernel: + fallback-package: linux-image-generic + package: linux-image-generic-lts-xenial + mapping: + - xenial: + - 4.4.0: -my-custom-kernel + + +kexec +~~~~~ +Curtin can use kexec to "reboot" into the target OS. + +**mode**: ** + +Enable rebooting with kexec. + +**Example**:: + + kexec: on + + +multipath +~~~~~~~~~ +Curtin will detect and autoconfigure multipath by default to enable +boot for systems with multipath. Curtin does not apply any advanced +configuration or tuning, rather it uses distro defaults and provides +enough configuration to enable booting. + +**mode**: *<['auto', ['disabled']>* + +Defaults to auto which will configure enough to enable booting on multipath +devices. Disabled will prevent curtin from installing or configuring +multipath. + +**overwrite_bindings**: ** + +If ``overwrite_bindings`` is True then Curtin will generate new bindings +file for multipath, overriding any existing binding in the target image. + +**Example**:: + + multipath: + mode: auto + overwrite_bindings: True + + +network +~~~~~~~ +Configure networking (see Networking section for details). + +**network_option_1**: *