From 28c18c19fc06d4b19b88fc3675bae6c92c5e8608 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Mon, 13 Dec 2010 21:36:18 +0000 Subject: [PATCH 01/46] Initial commit --- README.markdown | 59 ++ argparse.py | 2353 +++++++++++++++++++++++++++++++++++++++++++++++ cli53.py | 322 +++++++ 3 files changed, 2734 insertions(+) create mode 100644 README.markdown create mode 100644 argparse.py create mode 100755 cli53.py diff --git a/README.markdown b/README.markdown new file mode 100644 index 00000000..bdae03f3 --- /dev/null +++ b/README.markdown @@ -0,0 +1,59 @@ +cli53 - Command line script to administer the Amazon Route 53 dns service +========================================================================= + +Introduction +------------ +The latest Amazon service Route 53 is a great addition, but only has a rudimentary set of tools +available at the time of release. This script fills that gap until things have caught up. + +Features: +- create hosted zones + +- delete hosted zones + +- list hosted zones + +- import to BIND format + +- export to BIND format + +- create resource records + +- delete resource records + +- works with BIND format zone files we all know and love - no need to edit + XML! + +Getting Started +--------------- + +Create a hosted zone: + ./cli53.py create example.com + +Check what we've done: + ./cli53.py list + +Import a BIND zone file: + ./cli53.py import example.com --file zonefile.txt + +Manually create some records: + ./cli53.py rrcreate example.com www A 192.168.0.1 --ttl 3600 + ./cli53.py rrcreate example.com www A 192.168.0.2 --ttl 3600 --replace + ./cli53.py rrcreate example.com '' MX '10 192.168.0.1' '20 192.168.0.2' + +Export as a BIND zone file (useful for checking): + ./cli53.py export example.com + +Further documentation is available, e.g.: + ./cli53.py --help + ./cli53.py rrcreate --help + +Installation +------------ +There is no need to install, but you will need python, the latest boto library (from git) and dnspython: + + $ git clone boto && cd boto && python setup.py install + $ easy_install dnspython + +You need to set your Amazon credentials in the environment as AWS_ACCESS_KEY_ID +and AWS_SECRET_ACCESS_KEY. diff --git a/argparse.py b/argparse.py new file mode 100644 index 00000000..717b660b --- /dev/null +++ b/argparse.py @@ -0,0 +1,2353 @@ +# -*- coding: utf-8 -*- + +# Copyright © 2006-2009 Steven J. Bethard . +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Command-line parsing library + +This module is an optparse-inspired command-line parsing library that: + + - handles both optional and positional arguments + - produces highly informative usage messages + - supports parsers that dispatch to sub-parsers + +The following is a simple usage example that sums integers from the +command-line and writes the result to a file:: + + parser = argparse.ArgumentParser( + description='sum the integers at the command line') + parser.add_argument( + 'integers', metavar='int', nargs='+', type=int, + help='an integer to be summed') + parser.add_argument( + '--log', default=sys.stdout, type=argparse.FileType('w'), + help='the file where the sum should be written') + args = parser.parse_args() + args.log.write('%s' % sum(args.integers)) + args.log.close() + +The module contains the following public classes: + + - ArgumentParser -- The main entry point for command-line parsing. As the + example above shows, the add_argument() method is used to populate + the parser with actions for optional and positional arguments. Then + the parse_args() method is invoked to convert the args at the + command-line into an object with attributes. + + - ArgumentError -- The exception raised by ArgumentParser objects when + there are errors with the parser's actions. Errors raised while + parsing the command-line are caught by ArgumentParser and emitted + as command-line messages. + + - FileType -- A factory for defining types of files to be created. As the + example above shows, instances of FileType are typically passed as + the type= argument of add_argument() calls. + + - Action -- The base class for parser actions. Typically actions are + selected by passing strings like 'store_true' or 'append_const' to + the action= argument of add_argument(). However, for greater + customization of ArgumentParser actions, subclasses of Action may + be defined and passed as the action= argument. + + - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, + ArgumentDefaultsHelpFormatter -- Formatter classes which + may be passed as the formatter_class= argument to the + ArgumentParser constructor. HelpFormatter is the default, + RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser + not to change the formatting for help text, and + ArgumentDefaultsHelpFormatter adds information about argument defaults + to the help. + +All other classes in this module are considered implementation details. +(Also note that HelpFormatter and RawDescriptionHelpFormatter are only +considered public as object names -- the API of the formatter objects is +still considered an implementation detail.) +""" + +__version__ = '1.1' +__all__ = [ + 'ArgumentParser', + 'ArgumentError', + 'Namespace', + 'Action', + 'FileType', + 'HelpFormatter', + 'RawDescriptionHelpFormatter', + 'RawTextHelpFormatter', + 'ArgumentDefaultsHelpFormatter', +] + + +import copy as _copy +import os as _os +import re as _re +import sys as _sys +import textwrap as _textwrap + +from gettext import gettext as _ + +try: + _set = set +except NameError: + from sets import Set as _set + +try: + _basestring = basestring +except NameError: + _basestring = str + +try: + _sorted = sorted +except NameError: + + def _sorted(iterable, reverse=False): + result = list(iterable) + result.sort() + if reverse: + result.reverse() + return result + + +def _callable(obj): + return hasattr(obj, '__call__') or hasattr(obj, '__bases__') + +# silence Python 2.6 buggy warnings about Exception.message +if _sys.version_info[:2] == (2, 6): + import warnings + warnings.filterwarnings( + action='ignore', + message='BaseException.message has been deprecated as of Python 2.6', + category=DeprecationWarning, + module='argparse') + + +SUPPRESS = '==SUPPRESS==' + +OPTIONAL = '?' +ZERO_OR_MORE = '*' +ONE_OR_MORE = '+' +PARSER = 'A...' +REMAINDER = '...' + +# ============================= +# Utility functions and classes +# ============================= + +class _AttributeHolder(object): + """Abstract base class that provides __repr__. + + The __repr__ method returns a string in the format:: + ClassName(attr=name, attr=name, ...) + The attributes are determined either by a class-level attribute, + '_kwarg_names', or by inspecting the instance __dict__. + """ + + def __repr__(self): + type_name = type(self).__name__ + arg_strings = [] + for arg in self._get_args(): + arg_strings.append(repr(arg)) + for name, value in self._get_kwargs(): + arg_strings.append('%s=%r' % (name, value)) + return '%s(%s)' % (type_name, ', '.join(arg_strings)) + + def _get_kwargs(self): + return _sorted(self.__dict__.items()) + + def _get_args(self): + return [] + + +def _ensure_value(namespace, name, value): + if getattr(namespace, name, None) is None: + setattr(namespace, name, value) + return getattr(namespace, name) + + +# =============== +# Formatting Help +# =============== + +class HelpFormatter(object): + """Formatter for generating usage messages and argument help strings. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def __init__(self, + prog, + indent_increment=2, + max_help_position=24, + width=None): + + # default setting for width + if width is None: + try: + width = int(_os.environ['COLUMNS']) + except (KeyError, ValueError): + width = 80 + width -= 2 + + self._prog = prog + self._indent_increment = indent_increment + self._max_help_position = max_help_position + self._width = width + + self._current_indent = 0 + self._level = 0 + self._action_max_length = 0 + + self._root_section = self._Section(self, None) + self._current_section = self._root_section + + self._whitespace_matcher = _re.compile(r'\s+') + self._long_break_matcher = _re.compile(r'\n\n\n+') + + # =============================== + # Section and indentation methods + # =============================== + def _indent(self): + self._current_indent += self._indent_increment + self._level += 1 + + def _dedent(self): + self._current_indent -= self._indent_increment + assert self._current_indent >= 0, 'Indent decreased below 0.' + self._level -= 1 + + class _Section(object): + + def __init__(self, formatter, parent, heading=None): + self.formatter = formatter + self.parent = parent + self.heading = heading + self.items = [] + + def format_help(self): + # format the indented section + if self.parent is not None: + self.formatter._indent() + join = self.formatter._join_parts + for func, args in self.items: + func(*args) + item_help = join([func(*args) for func, args in self.items]) + if self.parent is not None: + self.formatter._dedent() + + # return nothing if the section was empty + if not item_help: + return '' + + # add the heading if the section was non-empty + if self.heading is not SUPPRESS and self.heading is not None: + current_indent = self.formatter._current_indent + heading = '%*s%s:\n' % (current_indent, '', self.heading) + else: + heading = '' + + # join the section-initial newline, the heading and the help + return join(['\n', heading, item_help, '\n']) + + def _add_item(self, func, args): + self._current_section.items.append((func, args)) + + # ======================== + # Message building methods + # ======================== + def start_section(self, heading): + self._indent() + section = self._Section(self, self._current_section, heading) + self._add_item(section.format_help, []) + self._current_section = section + + def end_section(self): + self._current_section = self._current_section.parent + self._dedent() + + def add_text(self, text): + if text is not SUPPRESS and text is not None: + self._add_item(self._format_text, [text]) + + def add_usage(self, usage, actions, groups, prefix=None): + if usage is not SUPPRESS: + args = usage, actions, groups, prefix + self._add_item(self._format_usage, args) + + def add_argument(self, action): + if action.help is not SUPPRESS: + + # find all invocations + get_invocation = self._format_action_invocation + invocations = [get_invocation(action)] + for subaction in self._iter_indented_subactions(action): + invocations.append(get_invocation(subaction)) + + # update the maximum item length + invocation_length = max([len(s) for s in invocations]) + action_length = invocation_length + self._current_indent + self._action_max_length = max(self._action_max_length, + action_length) + + # add the item to the list + self._add_item(self._format_action, [action]) + + def add_arguments(self, actions): + for action in actions: + self.add_argument(action) + + # ======================= + # Help-formatting methods + # ======================= + def format_help(self): + help = self._root_section.format_help() + if help: + help = self._long_break_matcher.sub('\n\n', help) + help = help.strip('\n') + '\n' + return help + + def _join_parts(self, part_strings): + return ''.join([part + for part in part_strings + if part and part is not SUPPRESS]) + + def _format_usage(self, usage, actions, groups, prefix): + if prefix is None: + prefix = _('usage: ') + + # if usage is specified, use that + if usage is not None: + usage = usage % dict(prog=self._prog) + + # if no optionals or positionals are available, usage is just prog + elif usage is None and not actions: + usage = '%(prog)s' % dict(prog=self._prog) + + # if optionals and positionals are available, calculate usage + elif usage is None: + prog = '%(prog)s' % dict(prog=self._prog) + + # split optionals from positionals + optionals = [] + positionals = [] + for action in actions: + if action.option_strings: + optionals.append(action) + else: + positionals.append(action) + + # build full usage string + format = self._format_actions_usage + action_usage = format(optionals + positionals, groups) + usage = ' '.join([s for s in [prog, action_usage] if s]) + + # wrap the usage parts if it's too long + text_width = self._width - self._current_indent + if len(prefix) + len(usage) > text_width: + + # break usage into wrappable parts + part_regexp = r'\(.*?\)+|\[.*?\]+|\S+' + opt_usage = format(optionals, groups) + pos_usage = format(positionals, groups) + opt_parts = _re.findall(part_regexp, opt_usage) + pos_parts = _re.findall(part_regexp, pos_usage) + assert ' '.join(opt_parts) == opt_usage + assert ' '.join(pos_parts) == pos_usage + + # helper for wrapping lines + def get_lines(parts, indent, prefix=None): + lines = [] + line = [] + if prefix is not None: + line_len = len(prefix) - 1 + else: + line_len = len(indent) - 1 + for part in parts: + if line_len + 1 + len(part) > text_width: + lines.append(indent + ' '.join(line)) + line = [] + line_len = len(indent) - 1 + line.append(part) + line_len += len(part) + 1 + if line: + lines.append(indent + ' '.join(line)) + if prefix is not None: + lines[0] = lines[0][len(indent):] + return lines + + # if prog is short, follow it with optionals or positionals + if len(prefix) + len(prog) <= 0.75 * text_width: + indent = ' ' * (len(prefix) + len(prog) + 1) + if opt_parts: + lines = get_lines([prog] + opt_parts, indent, prefix) + lines.extend(get_lines(pos_parts, indent)) + elif pos_parts: + lines = get_lines([prog] + pos_parts, indent, prefix) + else: + lines = [prog] + + # if prog is long, put it on its own line + else: + indent = ' ' * len(prefix) + parts = opt_parts + pos_parts + lines = get_lines(parts, indent) + if len(lines) > 1: + lines = [] + lines.extend(get_lines(opt_parts, indent)) + lines.extend(get_lines(pos_parts, indent)) + lines = [prog] + lines + + # join lines into usage + usage = '\n'.join(lines) + + # prefix with 'usage:' + return '%s%s\n\n' % (prefix, usage) + + def _format_actions_usage(self, actions, groups): + # find group indices and identify actions in groups + group_actions = _set() + inserts = {} + for group in groups: + try: + start = actions.index(group._group_actions[0]) + except ValueError: + continue + else: + end = start + len(group._group_actions) + if actions[start:end] == group._group_actions: + for action in group._group_actions: + group_actions.add(action) + if not group.required: + inserts[start] = '[' + inserts[end] = ']' + else: + inserts[start] = '(' + inserts[end] = ')' + for i in range(start + 1, end): + inserts[i] = '|' + + # collect all actions format strings + parts = [] + for i, action in enumerate(actions): + + # suppressed arguments are marked with None + # remove | separators for suppressed arguments + if action.help is SUPPRESS: + parts.append(None) + if inserts.get(i) == '|': + inserts.pop(i) + elif inserts.get(i + 1) == '|': + inserts.pop(i + 1) + + # produce all arg strings + elif not action.option_strings: + part = self._format_args(action, action.dest) + + # if it's in a group, strip the outer [] + if action in group_actions: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + + # add the action string to the list + parts.append(part) + + # produce the first way to invoke the option in brackets + else: + option_string = action.option_strings[0] + + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = '%s' % option_string + + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + part = '%s %s' % (option_string, args_string) + + # make it look optional if it's not required or in a group + if not action.required and action not in group_actions: + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + # insert things at the necessary indices + for i in _sorted(inserts, reverse=True): + parts[i:i] = [inserts[i]] + + # join all the action items with spaces + text = ' '.join([item for item in parts if item is not None]) + + # clean up separators for mutually exclusive groups + open = r'[\[(]' + close = r'[\])]' + text = _re.sub(r'(%s) ' % open, r'\1', text) + text = _re.sub(r' (%s)' % close, r'\1', text) + text = _re.sub(r'%s *%s' % (open, close), r'', text) + text = _re.sub(r'\(([^|]*)\)', r'\1', text) + text = text.strip() + + # return the text + return text + + def _format_text(self, text): + if '%(prog)' in text: + text = text % dict(prog=self._prog) + text_width = self._width - self._current_indent + indent = ' ' * self._current_indent + return self._fill_text(text, text_width, indent) + '\n\n' + + def _format_action(self, action): + # determine the required width and the entry label + help_position = min(self._action_max_length + 2, + self._max_help_position) + help_width = self._width - help_position + action_width = help_position - self._current_indent - 2 + action_header = self._format_action_invocation(action) + + # ho nelp; start on same line and add a final newline + if not action.help: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + + # short action name; start on the same line and pad two spaces + elif len(action_header) <= action_width: + tup = self._current_indent, '', action_width, action_header + action_header = '%*s%-*s ' % tup + indent_first = 0 + + # long action name; start on the next line + else: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + indent_first = help_position + + # collect the pieces of the action help + parts = [action_header] + + # if there was help for the action, add lines of help text + if action.help: + help_text = self._expand_help(action) + help_lines = self._split_lines(help_text, help_width) + parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) + for line in help_lines[1:]: + parts.append('%*s%s\n' % (help_position, '', line)) + + # or add a newline if the description doesn't end with one + elif not action_header.endswith('\n'): + parts.append('\n') + + # if there are any sub-actions, add their help as well + for subaction in self._iter_indented_subactions(action): + parts.append(self._format_action(subaction)) + + # return a single string + return self._join_parts(parts) + + def _format_action_invocation(self, action): + if not action.option_strings: + metavar, = self._metavar_formatter(action, action.dest)(1) + return metavar + + else: + parts = [] + + # if the Optional doesn't take a value, format is: + # -s, --long + if action.nargs == 0: + parts.extend(action.option_strings) + + # if the Optional takes a value, format is: + # -s ARGS, --long ARGS + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + for option_string in action.option_strings: + parts.append('%s %s' % (option_string, args_string)) + + return ', '.join(parts) + + def _metavar_formatter(self, action, default_metavar): + if action.metavar is not None: + result = action.metavar + elif action.choices is not None: + choice_strs = [str(choice) for choice in action.choices] + result = '{%s}' % ','.join(choice_strs) + else: + result = default_metavar + + def format(tuple_size): + if isinstance(result, tuple): + return result + else: + return (result, ) * tuple_size + return format + + def _format_args(self, action, default_metavar): + get_metavar = self._metavar_formatter(action, default_metavar) + if action.nargs is None: + result = '%s' % get_metavar(1) + elif action.nargs == OPTIONAL: + result = '[%s]' % get_metavar(1) + elif action.nargs == ZERO_OR_MORE: + result = '[%s [%s ...]]' % get_metavar(2) + elif action.nargs == ONE_OR_MORE: + result = '%s [%s ...]' % get_metavar(2) + elif action.nargs == REMAINDER: + result = '...' + elif action.nargs == PARSER: + result = '%s ...' % get_metavar(1) + else: + formats = ['%s' for _ in range(action.nargs)] + result = ' '.join(formats) % get_metavar(action.nargs) + return result + + def _expand_help(self, action): + params = dict(vars(action), prog=self._prog) + for name in list(params): + if params[name] is SUPPRESS: + del params[name] + for name in list(params): + if hasattr(params[name], '__name__'): + params[name] = params[name].__name__ + if params.get('choices') is not None: + choices_str = ', '.join([str(c) for c in params['choices']]) + params['choices'] = choices_str + return self._get_help_string(action) % params + + def _iter_indented_subactions(self, action): + try: + get_subactions = action._get_subactions + except AttributeError: + pass + else: + self._indent() + for subaction in get_subactions(): + yield subaction + self._dedent() + + def _split_lines(self, text, width): + text = self._whitespace_matcher.sub(' ', text).strip() + return _textwrap.wrap(text, width) + + def _fill_text(self, text, width, indent): + text = self._whitespace_matcher.sub(' ', text).strip() + return _textwrap.fill(text, width, initial_indent=indent, + subsequent_indent=indent) + + def _get_help_string(self, action): + return action.help + + +class RawDescriptionHelpFormatter(HelpFormatter): + """Help message formatter which retains any formatting in descriptions. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _fill_text(self, text, width, indent): + return ''.join([indent + line for line in text.splitlines(True)]) + + +class RawTextHelpFormatter(RawDescriptionHelpFormatter): + """Help message formatter which retains formatting of all help text. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _split_lines(self, text, width): + return text.splitlines() + + +class ArgumentDefaultsHelpFormatter(HelpFormatter): + """Help message formatter which adds default values to argument help. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _get_help_string(self, action): + help = action.help + if '%(default)' not in action.help: + if action.default is not SUPPRESS: + defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + help += ' (default: %(default)s)' + return help + + +# ===================== +# Options and Arguments +# ===================== + +def _get_action_name(argument): + if argument is None: + return None + elif argument.option_strings: + return '/'.join(argument.option_strings) + elif argument.metavar not in (None, SUPPRESS): + return argument.metavar + elif argument.dest not in (None, SUPPRESS): + return argument.dest + else: + return None + + +class ArgumentError(Exception): + """An error from creating or using an argument (optional or positional). + + The string value of this exception is the message, augmented with + information about the argument that caused it. + """ + + def __init__(self, argument, message): + self.argument_name = _get_action_name(argument) + self.message = message + + def __str__(self): + if self.argument_name is None: + format = '%(message)s' + else: + format = 'argument %(argument_name)s: %(message)s' + return format % dict(message=self.message, + argument_name=self.argument_name) + + +class ArgumentTypeError(Exception): + """An error from trying to convert a command line string to a type.""" + pass + + +# ============== +# Action classes +# ============== + +class Action(_AttributeHolder): + """Information about how to convert command line strings to Python objects. + + Action objects are used by an ArgumentParser to represent the information + needed to parse a single argument from one or more strings from the + command line. The keyword arguments to the Action constructor are also + all attributes of Action instances. + + Keyword Arguments: + + - option_strings -- A list of command-line option strings which + should be associated with this action. + + - dest -- The name of the attribute to hold the created object(s) + + - nargs -- The number of command-line arguments that should be + consumed. By default, one argument will be consumed and a single + value will be produced. Other values include: + - N (an integer) consumes N arguments (and produces a list) + - '?' consumes zero or one arguments + - '*' consumes zero or more arguments (and produces a list) + - '+' consumes one or more arguments (and produces a list) + Note that the difference between the default and nargs=1 is that + with the default, a single value will be produced, while with + nargs=1, a list containing a single value will be produced. + + - const -- The value to be produced if the option is specified and the + option uses an action that takes no values. + + - default -- The value to be produced if the option is not specified. + + - type -- The type which the command-line arguments should be converted + to, should be one of 'string', 'int', 'float', 'complex' or a + callable object that accepts a single string argument. If None, + 'string' is assumed. + + - choices -- A container of values that should be allowed. If not None, + after a command-line argument has been converted to the appropriate + type, an exception will be raised if it is not a member of this + collection. + + - required -- True if the action must always be specified at the + command line. This is only meaningful for optional command-line + arguments. + + - help -- The help string describing the argument. + + - metavar -- The name to be used for the option's argument with the + help string. If None, the 'dest' value will be used as the name. + """ + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + self.option_strings = option_strings + self.dest = dest + self.nargs = nargs + self.const = const + self.default = default + self.type = type + self.choices = choices + self.required = required + self.help = help + self.metavar = metavar + + def _get_kwargs(self): + names = [ + 'option_strings', + 'dest', + 'nargs', + 'const', + 'default', + 'type', + 'choices', + 'help', + 'metavar', + ] + return [(name, getattr(self, name)) for name in names] + + def __call__(self, parser, namespace, values, option_string=None): + raise NotImplementedError(_('.__call__() not defined')) + + +class _StoreAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for store actions must be > 0; if you ' + 'have nothing to store, actions such as store ' + 'true or store const may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_StoreAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + + +class _StoreConstAction(Action): + + def __init__(self, + option_strings, + dest, + const, + default=None, + required=False, + help=None, + metavar=None): + super(_StoreConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, self.const) + + +class _StoreTrueAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=False, + required=False, + help=None): + super(_StoreTrueAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=True, + default=default, + required=required, + help=help) + + +class _StoreFalseAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=True, + required=False, + help=None): + super(_StoreFalseAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=False, + default=default, + required=required, + help=help) + + +class _AppendAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for append actions must be > 0; if arg ' + 'strings are not supplying the value to append, ' + 'the append const action may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_AppendAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = _copy.copy(_ensure_value(namespace, self.dest, [])) + items.append(values) + setattr(namespace, self.dest, items) + + +class _AppendConstAction(Action): + + def __init__(self, + option_strings, + dest, + const, + default=None, + required=False, + help=None, + metavar=None): + super(_AppendConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = _copy.copy(_ensure_value(namespace, self.dest, [])) + items.append(self.const) + setattr(namespace, self.dest, items) + + +class _CountAction(Action): + + def __init__(self, + option_strings, + dest, + default=None, + required=False, + help=None): + super(_CountAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + new_count = _ensure_value(namespace, self.dest, 0) + 1 + setattr(namespace, self.dest, new_count) + + +class _HelpAction(Action): + + def __init__(self, + option_strings, + dest=SUPPRESS, + default=SUPPRESS, + help=None): + super(_HelpAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + parser.print_help() + parser.exit() + + +class _VersionAction(Action): + + def __init__(self, + option_strings, + version=None, + dest=SUPPRESS, + default=SUPPRESS, + help=None): + super(_VersionAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + self.version = version + + def __call__(self, parser, namespace, values, option_string=None): + version = self.version + if version is None: + version = parser.version + formatter = parser._get_formatter() + formatter.add_text(version) + parser.exit(message=formatter.format_help()) + + +class _SubParsersAction(Action): + + class _ChoicesPseudoAction(Action): + + def __init__(self, name, help): + sup = super(_SubParsersAction._ChoicesPseudoAction, self) + sup.__init__(option_strings=[], dest=name, help=help) + + def __init__(self, + option_strings, + prog, + parser_class, + dest=SUPPRESS, + help=None, + metavar=None): + + self._prog_prefix = prog + self._parser_class = parser_class + self._name_parser_map = {} + self._choices_actions = [] + + super(_SubParsersAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=PARSER, + choices=self._name_parser_map, + help=help, + metavar=metavar) + + def add_parser(self, name, **kwargs): + # set prog from the existing prefix + if kwargs.get('prog') is None: + kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + + # create a pseudo-action to hold the choice help + if 'help' in kwargs: + help = kwargs.pop('help') + choice_action = self._ChoicesPseudoAction(name, help) + self._choices_actions.append(choice_action) + + # create the parser and add it to the map + parser = self._parser_class(**kwargs) + self._name_parser_map[name] = parser + return parser + + def _get_subactions(self): + return self._choices_actions + + def __call__(self, parser, namespace, values, option_string=None): + parser_name = values[0] + arg_strings = values[1:] + + # set the parser name if requested + if self.dest is not SUPPRESS: + setattr(namespace, self.dest, parser_name) + + # select the parser + try: + parser = self._name_parser_map[parser_name] + except KeyError: + tup = parser_name, ', '.join(self._name_parser_map) + msg = _('unknown parser %r (choices: %s)' % tup) + raise ArgumentError(self, msg) + + # parse all the remaining options into the namespace + parser.parse_args(arg_strings, namespace) + + +# ============== +# Type classes +# ============== + +class FileType(object): + """Factory for creating file object types + + Instances of FileType are typically passed as type= arguments to the + ArgumentParser add_argument() method. + + Keyword Arguments: + - mode -- A string indicating how the file is to be opened. Accepts the + same values as the builtin open() function. + - bufsize -- The file's desired buffer size. Accepts the same values as + the builtin open() function. + """ + + def __init__(self, mode='r', bufsize=None): + self._mode = mode + self._bufsize = bufsize + + def __call__(self, string): + # the special argument "-" means sys.std{in,out} + if string == '-': + if 'r' in self._mode: + return _sys.stdin + elif 'w' in self._mode: + return _sys.stdout + else: + msg = _('argument "-" with mode %r' % self._mode) + raise ValueError(msg) + + # all other arguments are used as file names + if self._bufsize: + return open(string, self._mode, self._bufsize) + else: + return open(string, self._mode) + + def __repr__(self): + args = [self._mode, self._bufsize] + args_str = ', '.join([repr(arg) for arg in args if arg is not None]) + return '%s(%s)' % (type(self).__name__, args_str) + +# =========================== +# Optional and Positional Parsing +# =========================== + +class Namespace(_AttributeHolder): + """Simple object for storing attributes. + + Implements equality by attribute names and values, and provides a simple + string representation. + """ + + def __init__(self, **kwargs): + for name in kwargs: + setattr(self, name, kwargs[name]) + + def __eq__(self, other): + return vars(self) == vars(other) + + def __ne__(self, other): + return not (self == other) + + def __contains__(self, key): + return key in self.__dict__ + + +class _ActionsContainer(object): + + def __init__(self, + description, + prefix_chars, + argument_default, + conflict_handler): + super(_ActionsContainer, self).__init__() + + self.description = description + self.argument_default = argument_default + self.prefix_chars = prefix_chars + self.conflict_handler = conflict_handler + + # set up registries + self._registries = {} + + # register actions + self.register('action', None, _StoreAction) + self.register('action', 'store', _StoreAction) + self.register('action', 'store_const', _StoreConstAction) + self.register('action', 'store_true', _StoreTrueAction) + self.register('action', 'store_false', _StoreFalseAction) + self.register('action', 'append', _AppendAction) + self.register('action', 'append_const', _AppendConstAction) + self.register('action', 'count', _CountAction) + self.register('action', 'help', _HelpAction) + self.register('action', 'version', _VersionAction) + self.register('action', 'parsers', _SubParsersAction) + + # raise an exception if the conflict handler is invalid + self._get_handler() + + # action storage + self._actions = [] + self._option_string_actions = {} + + # groups + self._action_groups = [] + self._mutually_exclusive_groups = [] + + # defaults storage + self._defaults = {} + + # determines whether an "option" looks like a negative number + self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + + # whether or not there are any optionals that look like negative + # numbers -- uses a list so it can be shared and edited + self._has_negative_number_optionals = [] + + # ==================== + # Registration methods + # ==================== + def register(self, registry_name, value, object): + registry = self._registries.setdefault(registry_name, {}) + registry[value] = object + + def _registry_get(self, registry_name, value, default=None): + return self._registries[registry_name].get(value, default) + + # ================================== + # Namespace default accessor methods + # ================================== + def set_defaults(self, **kwargs): + self._defaults.update(kwargs) + + # if these defaults match any existing arguments, replace + # the previous default on the object with the new one + for action in self._actions: + if action.dest in kwargs: + action.default = kwargs[action.dest] + + def get_default(self, dest): + for action in self._actions: + if action.dest == dest and action.default is not None: + return action.default + return self._defaults.get(dest, None) + + + # ======================= + # Adding argument actions + # ======================= + def add_argument(self, *args, **kwargs): + """ + add_argument(dest, ..., name=value, ...) + add_argument(option_string, option_string, ..., name=value, ...) + """ + + # if no positional args are supplied or only one is supplied and + # it doesn't look like an option string, parse a positional + # argument + chars = self.prefix_chars + if not args or len(args) == 1 and args[0][0] not in chars: + if args and 'dest' in kwargs: + raise ValueError('dest supplied twice for positional argument') + kwargs = self._get_positional_kwargs(*args, **kwargs) + + # otherwise, we're adding an optional argument + else: + kwargs = self._get_optional_kwargs(*args, **kwargs) + + # if no default was supplied, use the parser-level default + if 'default' not in kwargs: + dest = kwargs['dest'] + if dest in self._defaults: + kwargs['default'] = self._defaults[dest] + elif self.argument_default is not None: + kwargs['default'] = self.argument_default + + # create the action object, and add it to the parser + action_class = self._pop_action_class(kwargs) + if not _callable(action_class): + raise ValueError('unknown action "%s"' % action_class) + action = action_class(**kwargs) + + # raise an error if the action type is not callable + type_func = self._registry_get('type', action.type, action.type) + if not _callable(type_func): + raise ValueError('%r is not callable' % type_func) + + return self._add_action(action) + + def add_argument_group(self, *args, **kwargs): + group = _ArgumentGroup(self, *args, **kwargs) + self._action_groups.append(group) + return group + + def add_mutually_exclusive_group(self, **kwargs): + group = _MutuallyExclusiveGroup(self, **kwargs) + self._mutually_exclusive_groups.append(group) + return group + + def _add_action(self, action): + # resolve any conflicts + self._check_conflict(action) + + # add to actions list + self._actions.append(action) + action.container = self + + # index the action by any option strings it has + for option_string in action.option_strings: + self._option_string_actions[option_string] = action + + # set the flag if any option strings look like negative numbers + for option_string in action.option_strings: + if self._negative_number_matcher.match(option_string): + if not self._has_negative_number_optionals: + self._has_negative_number_optionals.append(True) + + # return the created action + return action + + def _remove_action(self, action): + self._actions.remove(action) + + def _add_container_actions(self, container): + # collect groups by titles + title_group_map = {} + for group in self._action_groups: + if group.title in title_group_map: + msg = _('cannot merge actions - two groups are named %r') + raise ValueError(msg % (group.title)) + title_group_map[group.title] = group + + # map each action to its group + group_map = {} + for group in container._action_groups: + + # if a group with the title exists, use that, otherwise + # create a new group matching the container's group + if group.title not in title_group_map: + title_group_map[group.title] = self.add_argument_group( + title=group.title, + description=group.description, + conflict_handler=group.conflict_handler) + + # map the actions to their new group + for action in group._group_actions: + group_map[action] = title_group_map[group.title] + + # add container's mutually exclusive groups + # NOTE: if add_mutually_exclusive_group ever gains title= and + # description= then this code will need to be expanded as above + for group in container._mutually_exclusive_groups: + mutex_group = self.add_mutually_exclusive_group( + required=group.required) + + # map the actions to their new mutex group + for action in group._group_actions: + group_map[action] = mutex_group + + # add all actions to this container or their group + for action in container._actions: + group_map.get(action, self)._add_action(action) + + def _get_positional_kwargs(self, dest, **kwargs): + # make sure required is not specified + if 'required' in kwargs: + msg = _("'required' is an invalid argument for positionals") + raise TypeError(msg) + + # mark positional arguments as required if at least one is + # always required + if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: + kwargs['required'] = True + if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: + kwargs['required'] = True + + # return the keyword arguments with no option strings + return dict(kwargs, dest=dest, option_strings=[]) + + def _get_optional_kwargs(self, *args, **kwargs): + # determine short and long option strings + option_strings = [] + long_option_strings = [] + for option_string in args: + # error on strings that don't start with an appropriate prefix + if not option_string[0] in self.prefix_chars: + msg = _('invalid option string %r: ' + 'must start with a character %r') + tup = option_string, self.prefix_chars + raise ValueError(msg % tup) + + # strings starting with two prefix characters are long options + option_strings.append(option_string) + if option_string[0] in self.prefix_chars: + if len(option_string) > 1: + if option_string[1] in self.prefix_chars: + long_option_strings.append(option_string) + + # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' + dest = kwargs.pop('dest', None) + if dest is None: + if long_option_strings: + dest_option_string = long_option_strings[0] + else: + dest_option_string = option_strings[0] + dest = dest_option_string.lstrip(self.prefix_chars) + if not dest: + msg = _('dest= is required for options like %r') + raise ValueError(msg % option_string) + dest = dest.replace('-', '_') + + # return the updated keyword arguments + return dict(kwargs, dest=dest, option_strings=option_strings) + + def _pop_action_class(self, kwargs, default=None): + action = kwargs.pop('action', default) + return self._registry_get('action', action, action) + + def _get_handler(self): + # determine function from conflict handler string + handler_func_name = '_handle_conflict_%s' % self.conflict_handler + try: + return getattr(self, handler_func_name) + except AttributeError: + msg = _('invalid conflict_resolution value: %r') + raise ValueError(msg % self.conflict_handler) + + def _check_conflict(self, action): + + # find all options that conflict with this option + confl_optionals = [] + for option_string in action.option_strings: + if option_string in self._option_string_actions: + confl_optional = self._option_string_actions[option_string] + confl_optionals.append((option_string, confl_optional)) + + # resolve any conflicts + if confl_optionals: + conflict_handler = self._get_handler() + conflict_handler(action, confl_optionals) + + def _handle_conflict_error(self, action, conflicting_actions): + message = _('conflicting option string(s): %s') + conflict_string = ', '.join([option_string + for option_string, action + in conflicting_actions]) + raise ArgumentError(action, message % conflict_string) + + def _handle_conflict_resolve(self, action, conflicting_actions): + + # remove all conflicting options + for option_string, action in conflicting_actions: + + # remove the conflicting option + action.option_strings.remove(option_string) + self._option_string_actions.pop(option_string, None) + + # if the option now has no option string, remove it from the + # container holding it + if not action.option_strings: + action.container._remove_action(action) + + +class _ArgumentGroup(_ActionsContainer): + + def __init__(self, container, title=None, description=None, **kwargs): + # add any missing keyword arguments by checking the container + update = kwargs.setdefault + update('conflict_handler', container.conflict_handler) + update('prefix_chars', container.prefix_chars) + update('argument_default', container.argument_default) + super_init = super(_ArgumentGroup, self).__init__ + super_init(description=description, **kwargs) + + # group attributes + self.title = title + self._group_actions = [] + + # share most attributes with the container + self._registries = container._registries + self._actions = container._actions + self._option_string_actions = container._option_string_actions + self._defaults = container._defaults + self._has_negative_number_optionals = \ + container._has_negative_number_optionals + + def _add_action(self, action): + action = super(_ArgumentGroup, self)._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + super(_ArgumentGroup, self)._remove_action(action) + self._group_actions.remove(action) + + +class _MutuallyExclusiveGroup(_ArgumentGroup): + + def __init__(self, container, required=False): + super(_MutuallyExclusiveGroup, self).__init__(container) + self.required = required + self._container = container + + def _add_action(self, action): + if action.required: + msg = _('mutually exclusive arguments must be optional') + raise ValueError(msg) + action = self._container._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + self._container._remove_action(action) + self._group_actions.remove(action) + + +class ArgumentParser(_AttributeHolder, _ActionsContainer): + """Object for parsing command line strings into Python objects. + + Keyword Arguments: + - prog -- The name of the program (default: sys.argv[0]) + - usage -- A usage message (default: auto-generated from arguments) + - description -- A description of what the program does + - epilog -- Text following the argument descriptions + - parents -- Parsers whose arguments should be copied into this one + - formatter_class -- HelpFormatter class for printing help messages + - prefix_chars -- Characters that prefix optional arguments + - fromfile_prefix_chars -- Characters that prefix files containing + additional arguments + - argument_default -- The default value for all arguments + - conflict_handler -- String indicating how to handle conflicts + - add_help -- Add a -h/-help option + """ + + def __init__(self, + prog=None, + usage=None, + description=None, + epilog=None, + version=None, + parents=[], + formatter_class=HelpFormatter, + prefix_chars='-', + fromfile_prefix_chars=None, + argument_default=None, + conflict_handler='error', + add_help=True): + + if version is not None: + import warnings + warnings.warn( + """The "version" argument to ArgumentParser is deprecated. """ + """Please use """ + """"add_argument(..., action='version', version="N", ...)" """ + """instead""", DeprecationWarning) + + superinit = super(ArgumentParser, self).__init__ + superinit(description=description, + prefix_chars=prefix_chars, + argument_default=argument_default, + conflict_handler=conflict_handler) + + # default setting for prog + if prog is None: + prog = _os.path.basename(_sys.argv[0]) + + self.prog = prog + self.usage = usage + self.epilog = epilog + self.version = version + self.formatter_class = formatter_class + self.fromfile_prefix_chars = fromfile_prefix_chars + self.add_help = add_help + + add_group = self.add_argument_group + self._positionals = add_group(_('positional arguments')) + self._optionals = add_group(_('optional arguments')) + self._subparsers = None + + # register types + def identity(string): + return string + self.register('type', None, identity) + + # add help and version arguments if necessary + # (using explicit default to override global argument_default) + if self.add_help: + self.add_argument( + '-h', '--help', action='help', default=SUPPRESS, + help=_('show this help message and exit')) + if self.version: + self.add_argument( + '-v', '--version', action='version', default=SUPPRESS, + version=self.version, + help=_("show program's version number and exit")) + + # add parent arguments and defaults + for parent in parents: + self._add_container_actions(parent) + try: + defaults = parent._defaults + except AttributeError: + pass + else: + self._defaults.update(defaults) + + # ======================= + # Pretty __repr__ methods + # ======================= + def _get_kwargs(self): + names = [ + 'prog', + 'usage', + 'description', + 'version', + 'formatter_class', + 'conflict_handler', + 'add_help', + ] + return [(name, getattr(self, name)) for name in names] + + # ================================== + # Optional/Positional adding methods + # ================================== + def add_subparsers(self, **kwargs): + if self._subparsers is not None: + self.error(_('cannot have multiple subparser arguments')) + + # add the parser class to the arguments if it's not present + kwargs.setdefault('parser_class', type(self)) + + if 'title' in kwargs or 'description' in kwargs: + title = _(kwargs.pop('title', 'subcommands')) + description = _(kwargs.pop('description', None)) + self._subparsers = self.add_argument_group(title, description) + else: + self._subparsers = self._positionals + + # prog defaults to the usage message of this parser, skipping + # optional arguments and with no "usage:" prefix + if kwargs.get('prog') is None: + formatter = self._get_formatter() + positionals = self._get_positional_actions() + groups = self._mutually_exclusive_groups + formatter.add_usage(self.usage, positionals, groups, '') + kwargs['prog'] = formatter.format_help().strip() + + # create the parsers action and add it to the positionals list + parsers_class = self._pop_action_class(kwargs, 'parsers') + action = parsers_class(option_strings=[], **kwargs) + self._subparsers._add_action(action) + + # return the created parsers action + return action + + def _add_action(self, action): + if action.option_strings: + self._optionals._add_action(action) + else: + self._positionals._add_action(action) + return action + + def _get_optional_actions(self): + return [action + for action in self._actions + if action.option_strings] + + def _get_positional_actions(self): + return [action + for action in self._actions + if not action.option_strings] + + # ===================================== + # Command line argument parsing methods + # ===================================== + def parse_args(self, args=None, namespace=None): + args, argv = self.parse_known_args(args, namespace) + if argv: + msg = _('unrecognized arguments: %s') + self.error(msg % ' '.join(argv)) + return args + + def parse_known_args(self, args=None, namespace=None): + # args default to the system args + if args is None: + args = _sys.argv[1:] + + # default Namespace built from parser defaults + if namespace is None: + namespace = Namespace() + + # add any action defaults that aren't present + for action in self._actions: + if action.dest is not SUPPRESS: + if not hasattr(namespace, action.dest): + if action.default is not SUPPRESS: + default = action.default + if isinstance(action.default, _basestring): + default = self._get_value(action, default) + setattr(namespace, action.dest, default) + + # add any parser defaults that aren't present + for dest in self._defaults: + if not hasattr(namespace, dest): + setattr(namespace, dest, self._defaults[dest]) + + # parse the arguments and exit if there are any errors + try: + return self._parse_known_args(args, namespace) + except ArgumentError: + err = _sys.exc_info()[1] + self.error(str(err)) + + def _parse_known_args(self, arg_strings, namespace): + # replace arg strings that are file references + if self.fromfile_prefix_chars is not None: + arg_strings = self._read_args_from_files(arg_strings) + + # map all mutually exclusive arguments to the other arguments + # they can't occur with + action_conflicts = {} + for mutex_group in self._mutually_exclusive_groups: + group_actions = mutex_group._group_actions + for i, mutex_action in enumerate(mutex_group._group_actions): + conflicts = action_conflicts.setdefault(mutex_action, []) + conflicts.extend(group_actions[:i]) + conflicts.extend(group_actions[i + 1:]) + + # find all option indices, and determine the arg_string_pattern + # which has an 'O' if there is an option at an index, + # an 'A' if there is an argument, or a '-' if there is a '--' + option_string_indices = {} + arg_string_pattern_parts = [] + arg_strings_iter = iter(arg_strings) + for i, arg_string in enumerate(arg_strings_iter): + + # all args after -- are non-options + if arg_string == '--': + arg_string_pattern_parts.append('-') + for arg_string in arg_strings_iter: + arg_string_pattern_parts.append('A') + + # otherwise, add the arg to the arg strings + # and note the index if it was an option + else: + option_tuple = self._parse_optional(arg_string) + if option_tuple is None: + pattern = 'A' + else: + option_string_indices[i] = option_tuple + pattern = 'O' + arg_string_pattern_parts.append(pattern) + + # join the pieces together to form the pattern + arg_strings_pattern = ''.join(arg_string_pattern_parts) + + # converts arg strings to the appropriate and then takes the action + seen_actions = _set() + seen_non_default_actions = _set() + + def take_action(action, argument_strings, option_string=None): + seen_actions.add(action) + argument_values = self._get_values(action, argument_strings) + + # error if this argument is not allowed with other previously + # seen arguments, assuming that actions that use the default + # value don't really count as "present" + if argument_values is not action.default: + seen_non_default_actions.add(action) + for conflict_action in action_conflicts.get(action, []): + if conflict_action in seen_non_default_actions: + msg = _('not allowed with argument %s') + action_name = _get_action_name(conflict_action) + raise ArgumentError(action, msg % action_name) + + # take the action if we didn't receive a SUPPRESS value + # (e.g. from a default) + if argument_values is not SUPPRESS: + action(self, namespace, argument_values, option_string) + + # function to convert arg_strings into an optional action + def consume_optional(start_index): + + # get the optional identified at this index + option_tuple = option_string_indices[start_index] + action, option_string, explicit_arg = option_tuple + + # identify additional optionals in the same arg string + # (e.g. -xyz is the same as -x -y -z if no args are required) + match_argument = self._match_argument + action_tuples = [] + while True: + + # if we found no optional action, skip it + if action is None: + extras.append(arg_strings[start_index]) + return start_index + 1 + + # if there is an explicit argument, try to match the + # optional's string arguments to only this + if explicit_arg is not None: + arg_count = match_argument(action, 'A') + + # if the action is a single-dash option and takes no + # arguments, try to parse more single-dash options out + # of the tail of the option string + chars = self.prefix_chars + if arg_count == 0 and option_string[1] not in chars: + action_tuples.append((action, [], option_string)) + for char in self.prefix_chars: + option_string = char + explicit_arg[0] + explicit_arg = explicit_arg[1:] or None + optionals_map = self._option_string_actions + if option_string in optionals_map: + action = optionals_map[option_string] + break + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if the action expect exactly one argument, we've + # successfully matched the option; exit the loop + elif arg_count == 1: + stop = start_index + 1 + args = [explicit_arg] + action_tuples.append((action, args, option_string)) + break + + # error if a double-dash option did not use the + # explicit argument + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if there is no explicit argument, try to match the + # optional's string arguments with the following strings + # if successful, exit the loop + else: + start = start_index + 1 + selected_patterns = arg_strings_pattern[start:] + arg_count = match_argument(action, selected_patterns) + stop = start + arg_count + args = arg_strings[start:stop] + action_tuples.append((action, args, option_string)) + break + + # add the Optional to the list and return the index at which + # the Optional's string args stopped + assert action_tuples + for action, args, option_string in action_tuples: + take_action(action, args, option_string) + return stop + + # the list of Positionals left to be parsed; this is modified + # by consume_positionals() + positionals = self._get_positional_actions() + + # function to convert arg_strings into positional actions + def consume_positionals(start_index): + # match as many Positionals as possible + match_partial = self._match_arguments_partial + selected_pattern = arg_strings_pattern[start_index:] + arg_counts = match_partial(positionals, selected_pattern) + + # slice off the appropriate arg strings for each Positional + # and add the Positional and its args to the list + for action, arg_count in zip(positionals, arg_counts): + args = arg_strings[start_index: start_index + arg_count] + start_index += arg_count + take_action(action, args) + + # slice off the Positionals that we just parsed and return the + # index at which the Positionals' string args stopped + positionals[:] = positionals[len(arg_counts):] + return start_index + + # consume Positionals and Optionals alternately, until we have + # passed the last option string + extras = [] + start_index = 0 + if option_string_indices: + max_option_string_index = max(option_string_indices) + else: + max_option_string_index = -1 + while start_index <= max_option_string_index: + + # consume any Positionals preceding the next option + next_option_string_index = min([ + index + for index in option_string_indices + if index >= start_index]) + if start_index != next_option_string_index: + positionals_end_index = consume_positionals(start_index) + + # only try to parse the next optional if we didn't consume + # the option string during the positionals parsing + if positionals_end_index > start_index: + start_index = positionals_end_index + continue + else: + start_index = positionals_end_index + + # if we consumed all the positionals we could and we're not + # at the index of an option string, there were extra arguments + if start_index not in option_string_indices: + strings = arg_strings[start_index:next_option_string_index] + extras.extend(strings) + start_index = next_option_string_index + + # consume the next optional and any arguments for it + start_index = consume_optional(start_index) + + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index) + + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + + # if we didn't use all the Positional objects, there were too few + # arg strings supplied. + if positionals: + self.error(_('too few arguments')) + + # make sure all required actions were present + for action in self._actions: + if action.required: + if action not in seen_actions: + name = _get_action_name(action) + self.error(_('argument %s is required') % name) + + # make sure all required groups had one option present + for group in self._mutually_exclusive_groups: + if group.required: + for action in group._group_actions: + if action in seen_non_default_actions: + break + + # if no actions were used, report the error + else: + names = [_get_action_name(action) + for action in group._group_actions + if action.help is not SUPPRESS] + msg = _('one of the arguments %s is required') + self.error(msg % ' '.join(names)) + + # return the updated namespace and the extra arguments + return namespace, extras + + def _read_args_from_files(self, arg_strings): + # expand arguments referencing files + new_arg_strings = [] + for arg_string in arg_strings: + + # for regular arguments, just add them back into the list + if arg_string[0] not in self.fromfile_prefix_chars: + new_arg_strings.append(arg_string) + + # replace arguments referencing files with the file content + else: + try: + args_file = open(arg_string[1:]) + try: + arg_strings = [] + for arg_line in args_file.read().splitlines(): + for arg in self.convert_arg_line_to_args(arg_line): + arg_strings.append(arg) + arg_strings = self._read_args_from_files(arg_strings) + new_arg_strings.extend(arg_strings) + finally: + args_file.close() + except IOError: + err = _sys.exc_info()[1] + self.error(str(err)) + + # return the modified argument list + return new_arg_strings + + def convert_arg_line_to_args(self, arg_line): + return [arg_line] + + def _match_argument(self, action, arg_strings_pattern): + # match the pattern for this action to the arg strings + nargs_pattern = self._get_nargs_pattern(action) + match = _re.match(nargs_pattern, arg_strings_pattern) + + # raise an exception if we weren't able to find a match + if match is None: + nargs_errors = { + None: _('expected one argument'), + OPTIONAL: _('expected at most one argument'), + ONE_OR_MORE: _('expected at least one argument'), + } + default = _('expected %s argument(s)') % action.nargs + msg = nargs_errors.get(action.nargs, default) + raise ArgumentError(action, msg) + + # return the number of arguments matched + return len(match.group(1)) + + def _match_arguments_partial(self, actions, arg_strings_pattern): + # progressively shorten the actions list by slicing off the + # final actions until we find a match + result = [] + for i in range(len(actions), 0, -1): + actions_slice = actions[:i] + pattern = ''.join([self._get_nargs_pattern(action) + for action in actions_slice]) + match = _re.match(pattern, arg_strings_pattern) + if match is not None: + result.extend([len(string) for string in match.groups()]) + break + + # return the list of arg string counts + return result + + def _parse_optional(self, arg_string): + # if it's an empty string, it was meant to be a positional + if not arg_string: + return None + + # if it doesn't start with a prefix, it was meant to be positional + if not arg_string[0] in self.prefix_chars: + return None + + # if the option string is present in the parser, return the action + if arg_string in self._option_string_actions: + action = self._option_string_actions[arg_string] + return action, arg_string, None + + # if it's just a single character, it was meant to be positional + if len(arg_string) == 1: + return None + + # if the option string before the "=" is present, return the action + if '=' in arg_string: + option_string, explicit_arg = arg_string.split('=', 1) + if option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, explicit_arg + + # search through all possible prefixes of the option string + # and all actions in the parser for possible interpretations + option_tuples = self._get_option_tuples(arg_string) + + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, explicit_arg in option_tuples]) + tup = arg_string, options + self.error(_('ambiguous option: %s could match %s') % tup) + + # if exactly one action matched, this segmentation is good, + # so return the parsed action + elif len(option_tuples) == 1: + option_tuple, = option_tuples + return option_tuple + + # if it was not found as an option, but it looks like a negative + # number, it was meant to be positional + # unless there are negative-number-like options + if self._negative_number_matcher.match(arg_string): + if not self._has_negative_number_optionals: + return None + + # if it contains a space, it was meant to be a positional + if ' ' in arg_string: + return None + + # it was meant to be an optional but there is no such option + # in this parser (though it might be a valid option in a subparser) + return None, arg_string, None + + def _get_option_tuples(self, option_string): + result = [] + + # option strings starting with two prefix characters are only + # split at the '=' + chars = self.prefix_chars + if option_string[0] in chars and option_string[1] in chars: + if '=' in option_string: + option_prefix, explicit_arg = option_string.split('=', 1) + else: + option_prefix = option_string + explicit_arg = None + for option_string in self._option_string_actions: + if option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, explicit_arg + result.append(tup) + + # single character options can be concatenated with their arguments + # but multiple character options always have to have their argument + # separate + elif option_string[0] in chars and option_string[1] not in chars: + option_prefix = option_string + explicit_arg = None + short_option_prefix = option_string[:2] + short_explicit_arg = option_string[2:] + + for option_string in self._option_string_actions: + if option_string == short_option_prefix: + action = self._option_string_actions[option_string] + tup = action, option_string, short_explicit_arg + result.append(tup) + elif option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, explicit_arg + result.append(tup) + + # shouldn't ever get here + else: + self.error(_('unexpected option string: %s') % option_string) + + # return the collected option tuples + return result + + def _get_nargs_pattern(self, action): + # in all examples below, we have to allow for '--' args + # which are represented as '-' in the pattern + nargs = action.nargs + + # the default (None) is assumed to be a single argument + if nargs is None: + nargs_pattern = '(-*A-*)' + + # allow zero or one arguments + elif nargs == OPTIONAL: + nargs_pattern = '(-*A?-*)' + + # allow zero or more arguments + elif nargs == ZERO_OR_MORE: + nargs_pattern = '(-*[A-]*)' + + # allow one or more arguments + elif nargs == ONE_OR_MORE: + nargs_pattern = '(-*A[A-]*)' + + # allow any number of options or arguments + elif nargs == REMAINDER: + nargs_pattern = '([-AO]*)' + + # allow one argument followed by any number of options or arguments + elif nargs == PARSER: + nargs_pattern = '(-*A[-AO]*)' + + # all others should be integers + else: + nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) + + # if this is an optional action, -- is not allowed + if action.option_strings: + nargs_pattern = nargs_pattern.replace('-*', '') + nargs_pattern = nargs_pattern.replace('-', '') + + # return the pattern + return nargs_pattern + + # ======================== + # Value conversion methods + # ======================== + def _get_values(self, action, arg_strings): + # for everything but PARSER args, strip out '--' + if action.nargs not in [PARSER, REMAINDER]: + arg_strings = [s for s in arg_strings if s != '--'] + + # optional argument produces a default when not present + if not arg_strings and action.nargs == OPTIONAL: + if action.option_strings: + value = action.const + else: + value = action.default + if isinstance(value, _basestring): + value = self._get_value(action, value) + self._check_value(action, value) + + # when nargs='*' on a positional, if there were no command-line + # args, use the default if it is anything other than None + elif (not arg_strings and action.nargs == ZERO_OR_MORE and + not action.option_strings): + if action.default is not None: + value = action.default + else: + value = arg_strings + self._check_value(action, value) + + # single argument or optional argument produces a single value + elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: + arg_string, = arg_strings + value = self._get_value(action, arg_string) + self._check_value(action, value) + + # REMAINDER arguments convert all values, checking none + elif action.nargs == REMAINDER: + value = [self._get_value(action, v) for v in arg_strings] + + # PARSER arguments convert all values, but check only the first + elif action.nargs == PARSER: + value = [self._get_value(action, v) for v in arg_strings] + self._check_value(action, value[0]) + + # all other types of nargs produce a list + else: + value = [self._get_value(action, v) for v in arg_strings] + for v in value: + self._check_value(action, v) + + # return the converted value + return value + + def _get_value(self, action, arg_string): + type_func = self._registry_get('type', action.type, action.type) + if not _callable(type_func): + msg = _('%r is not callable') + raise ArgumentError(action, msg % type_func) + + # convert the value to the appropriate type + try: + result = type_func(arg_string) + + # ArgumentTypeErrors indicate errors + except ArgumentTypeError: + name = getattr(action.type, '__name__', repr(action.type)) + msg = str(_sys.exc_info()[1]) + raise ArgumentError(action, msg) + + # TypeErrors or ValueErrors also indicate errors + except (TypeError, ValueError): + name = getattr(action.type, '__name__', repr(action.type)) + msg = _('invalid %s value: %r') + raise ArgumentError(action, msg % (name, arg_string)) + + # return the converted value + return result + + def _check_value(self, action, value): + # converted value must be one of the choices (if specified) + if action.choices is not None and value not in action.choices: + tup = value, ', '.join(map(repr, action.choices)) + msg = _('invalid choice: %r (choose from %s)') % tup + raise ArgumentError(action, msg) + + # ======================= + # Help-formatting methods + # ======================= + def format_usage(self): + formatter = self._get_formatter() + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + return formatter.format_help() + + def format_help(self): + formatter = self._get_formatter() + + # usage + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + + # description + formatter.add_text(self.description) + + # positionals, optionals and user-defined groups + for action_group in self._action_groups: + formatter.start_section(action_group.title) + formatter.add_text(action_group.description) + formatter.add_arguments(action_group._group_actions) + formatter.end_section() + + # epilog + formatter.add_text(self.epilog) + + # determine help from format above + return formatter.format_help() + + def format_version(self): + import warnings + warnings.warn( + 'The format_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + formatter = self._get_formatter() + formatter.add_text(self.version) + return formatter.format_help() + + def _get_formatter(self): + return self.formatter_class(prog=self.prog) + + # ===================== + # Help-printing methods + # ===================== + def print_usage(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_usage(), file) + + def print_help(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_help(), file) + + def print_version(self, file=None): + import warnings + warnings.warn( + 'The print_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + self._print_message(self.format_version(), file) + + def _print_message(self, message, file=None): + if message: + if file is None: + file = _sys.stderr + file.write(message) + + # =============== + # Exiting methods + # =============== + def exit(self, status=0, message=None): + if message: + self._print_message(message, _sys.stderr) + _sys.exit(status) + + def error(self, message): + """error(message: string) + + Prints a usage message incorporating the message to stderr and + exits. + + If you override this in a subclass, it should not return -- it + should either exit or raise an exception. + """ + self.print_usage(_sys.stderr) + self.exit(2, _('%s: error: %s\n') % (self.prog, message)) diff --git a/cli53.py b/cli53.py new file mode 100755 index 00000000..9e304674 --- /dev/null +++ b/cli53.py @@ -0,0 +1,322 @@ +#!/usr/bin/python + +# cli53 +# Command line script to administer the Amazon Route 53 dns service + +import os, sys +import re +from cStringIO import StringIO + +# needs patched boto from github: http://github.com/barnybug/boto +try: + import boto.route53, boto.jsonresponse +except ImportError: + print "Please install latest boto:" + print "git clone boto && cd boto && python setup.py install" + sys.exit(-1) + +import argparse +from argparse import ArgumentError +from types import StringTypes +import xml.etree.ElementTree as et + +try: + import dns.zone, dns.rdataset, dns.node, dns.rdtypes, dns.rdataclass + import dns.rdtypes.ANY.CNAME, dns.rdtypes.ANY.SOA, dns.rdtypes.ANY.MX, dns.rdtypes.ANY.NS, dns.rdtypes.IN.A +except ImportError: + print "Please install dnspython:" + print "easy_install dnspython" + sys.exit(-1) + +if not (os.getenv('AWS_ACCESS_KEY_ID') and os.getenv('AWS_SECRET_ACCESS_KEY')): + print 'Please set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, e.g.:' + print 'export AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXX' + print 'export AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + sys.exit(-1) + +r53 = boto.route53.Route53Connection() + +def pprint(obj, findent='', indent=''): + if isinstance(obj, StringTypes): + print '%s%s' % (findent, obj) + elif isinstance(obj, boto.jsonresponse.Element): + i = findent + for k, v in obj.iteritems(): + if k in ('IsTruncated', 'MaxItems'): continue + if isinstance(v, StringTypes): + print '%s%s: %s' % (i, k, v) + else: + print '%s%s:' % (i, k) + pprint(v, indent+' ', indent+' ') + i = indent + elif isinstance(obj, (boto.jsonresponse.ListElement, list)): + i = findent + for v in obj: + pprint(v, i+'- ', i+' ') + i = indent + else: + raise ValueError, 'Cannot pprint type %s' % type(obj) + +def cmd_list(args): + ret = r53.get_all_hosted_zones() + pprint(ret.ListHostedZonesResponse) + +def cmd_info(args): + ret = r53.get_hosted_zone(args.zone) + pprint(ret.GetHostedZoneResponse) + +def text_element(parent, name, text): + el = et.SubElement(parent, name) + el.text = text + +class BindToR53Formatter(object): + def create_all(self, zone, exclude=()): + creates = [] + for name, node in zone.items(): + for rdataset in node.rdatasets: + if dns.rdatatype.to_text(rdataset.rdtype) not in exclude: + creates.append((name, rdataset)) + return self._xml_changes(zone, creates=creates) + + def create_record(self, zone, name, rdataset): + return self._xml_changes(zone, creates=[(name,rdataset)]) + + def delete_record(self, zone, name, rdataset): + return self._xml_changes(zone, deletes=[(name,rdataset)]) + + def replace_record(self, zone, name, rdataset_old, rdataset_new): + return self._xml_changes(zone, creates=[(name,rdataset_new)], deletes=[(name,rdataset_old)]) + + def _xml_changes(self, zone, creates=[], deletes=[]): + root = et.Element('ChangeResourceRecordSetsRequest', xmlns=boto.route53.Route53Connection.XMLNameSpace) + change_batch = et.SubElement(root, 'ChangeBatch') + changes = et.SubElement(change_batch, 'Changes') + + for chg, rdatasets in (('DELETE', deletes), ('CREATE', creates)): + for name, rdataset in rdatasets: + change = et.SubElement(changes, 'Change') + text_element(change, 'Action', chg) + rrset = et.SubElement(change, 'ResourceRecordSet') + text_element(rrset, 'Name', name.derelativize(zone.origin).to_text()) + text_element(rrset, 'Type', dns.rdatatype.to_text(rdataset.rdtype)) + text_element(rrset, 'TTL', str(rdataset.ttl)) + rrs = et.SubElement(rrset, 'ResourceRecords') + for rdtype in rdataset.items: + rr = et.SubElement(rrs, 'ResourceRecord') + text_element(rr, 'Value', rdtype.to_text()) + + out = StringIO() + et.ElementTree(root).write(out) + return out.getvalue() + +class R53ToBindFormatter(object): + def convert(self, info, xml): + origin = info.HostedZone.Name + z = dns.zone.Zone(dns.name.from_text(origin)) + + ns = boto.route53.Route53Connection.XMLNameSpace + tree = et.fromstring(xml) + + for rrsets in tree.findall("{%s}ResourceRecordSets" % ns): + for rrset in rrsets.findall("{%s}ResourceRecordSet" % ns): + name = rrset.find('{%s}Name' % ns).text + rtype = rrset.find('{%s}Type' % ns).text + ttl = int(rrset.find('{%s}TTL' % ns).text) + + values = [ rr.text for rr in rrset.findall('{%(ns)s}ResourceRecords/{%(ns)s}ResourceRecord/{%(ns)s}Value' % {'ns':ns}) ] + rdataset = _create_rdataset(rtype, ttl, values) + node = z.get_node(name, create=True) + node.replace_rdataset(rdataset) + + return z + +def _create_rdataset(rtype, ttl, values): + rdataset = dns.rdataset.Rdataset(dns.rdataclass.IN, dns.rdatatype.from_text(rtype)) + rdataset.ttl = ttl + for value in values: + rdtype = None + if rtype == 'A': + rdtype = dns.rdtypes.IN.A.A(dns.rdataclass.IN, dns.rdatatype.A, value) + elif rtype == 'CNAME': + rdtype = dns.rdtypes.ANY.CNAME.CNAME(dns.rdataclass.ANY, + dns.rdatatype.CNAME, dns.name.from_text(value)) + elif rtype == 'SOA': + mname, rname, serial, refresh, retry, expire, minimum = value.split() + mname = dns.name.from_text(mname) + rname = dns.name.from_text(rname) + serial = int(serial) + refresh = int(refresh) + retry = int(retry) + expire = int(expire) + minimum = int(minimum) + rdtype = dns.rdtypes.ANY.SOA.SOA(dns.rdataclass.ANY, dns.rdatatype.SOA, mname, rname, serial, refresh, retry, expire, minimum) + elif rtype == 'NS': + rdtype = dns.rdtypes.ANY.NS.NS(dns.rdataclass.ANY, dns.rdatatype.SOA, dns.name.from_text(value)) + elif rtype == 'MX': + pref, ex = value.split() + pref = int(pref) + rdtype = dns.rdtypes.ANY.MX.MX(dns.rdataclass.ANY, dns.rdatatype.SOA, pref, dns.name.from_text(ex)) + else: + raise ValueError, 'record type %s not handled' % rtype + rdataset.items.append(rdtype) + return rdataset + +def cmd_xml(args): + xml = r53.get_all_rrsets(args.zone) + print xml + +re_comment = re.compile('\S*;.*$') +re_dos = re.compile('\r\n$') +re_origin = re.compile(r'\$ORIGIN (\S+)') +def cmd_import(args): + text = [] + for line in args.file: + line = re_dos.sub('\n', line) + text.append(line) + text = ''.join(text) + + m = re_origin.search(text) + if not m: + raise Exception, 'Could not find origin' + origin = m.group(1) + + zone = dns.zone.from_text(text, origin=origin, check_origin=True) + f = BindToR53Formatter() + xml = f.create_all(zone, exclude=('SOA','NS')) + + ret = r53.change_rrsets(args.zone, xml) + pprint(ret.ChangeResourceRecordSetsResponse) + +re_zone_id = re.compile('^[A-Z0-9]{14}$') +def Zone(zone): + if re_zone_id.match(zone): + return zone + ret = r53.get_all_hosted_zones() + for hz in ret.ListHostedZonesResponse.HostedZones: + if hz.Name == zone or hz.Name == zone+'.': + return hz.Id.replace('/hostedzone/', '') + raise ArgumentError, 'Zone %s not found' % zone + +def _get_records(args): + info = r53.get_hosted_zone(args.zone) + xml = r53.get_all_rrsets(args.zone) + f = R53ToBindFormatter() + return f.convert(info.GetHostedZoneResponse, xml) + +def cmd_export(args): + zone = _get_records(args) + zone.to_file(sys.stdout) + +def cmd_create(args): + ret = r53.create_hosted_zone(args.zone) + pprint(ret.CreateHostedZoneResponse) + +def cmd_delete(args): + ret = r53.delete_hosted_zone(args.zone) + pprint(ret.DeleteHostedZoneResponse) + +def cmd_rrcreate(args): + zone = _get_records(args) + name = dns.name.from_text(args.rr, zone.origin) + rdataset = _create_rdataset(args.type, args.ttl, args.values) + + rdataset_old = None + node = zone.get_node(args.rr) + if node: + for rds in node.rdatasets: + if args.type == dns.rdatatype.to_text(rds.rdtype): + rdataset_old = rds + break + + if args.replace and rdataset_old: + xml = BindToR53Formatter().replace_record(zone, name, rdataset_old, rdataset) + else: + xml = BindToR53Formatter().create_record(zone, name, rdataset) + ret = r53.change_rrsets(args.zone, xml) + print 'Success' + pprint(ret.ChangeResourceRecordSetsResponse) + +def cmd_rrdelete(args): + zone = _get_records(args) + name = dns.name.from_text(args.rr, zone.origin) + + node = zone.get_node(args.rr) + if node: + if len(node.rdatasets) > 1 and not args.type: + rtypes = [ dns.rdatatype.to_text(rds.rdtype) for rds in node.rdatasets ] + print 'Ambigious record - several resource types for record %s found: %s' % (args.rr, ', '.join(rtypes)) + else: + rdataset = None + for rds in node.rdatasets: + if args.type == dns.rdatatype.to_text(rds.rdtype) or not args.type: + rdataset = rds + break + + if not rdataset: + print 'Record not found: %s, type: %s' % (args.rr, args.type) + return + + print 'Deleting %s %s...' % (args.rr, dns.rdatatype.to_text(rds.rdtype)) + + xml = BindToR53Formatter().delete_record(zone, name, rdataset) + ret = r53.change_rrsets(args.zone, xml) + print 'Success' + pprint(ret.ChangeResourceRecordSetsResponse) + else: + print 'Record not found: %s' % args.rr + +def main(): + connection = boto.route53.Route53Connection() + parser = argparse.ArgumentParser(description='route53 command line tool') + subparsers = parser.add_subparsers(help='sub-command help') + + supported_rtypes = ('A', 'CNAME', 'SOA', 'NS', 'MX') + + parser_list = subparsers.add_parser('list', help='list hosted zones') + parser_list.set_defaults(func=cmd_list) + + parser_list = subparsers.add_parser('info', help='get details of a hosted zone') + parser_list.add_argument('zone', type=Zone, help='zone name') + parser_list.set_defaults(func=cmd_info) + + parser_describe = subparsers.add_parser('xml', help='get the rrsets xml of a hosted zone') + parser_describe.add_argument('zone', type=Zone, help='zone name') + parser_describe.set_defaults(func=cmd_xml) + + parser_describe = subparsers.add_parser('export', help='export dns in bind format') + parser_describe.add_argument('zone', type=Zone, help='zone name') + parser_describe.set_defaults(func=cmd_export) + + parser_describe = subparsers.add_parser('import', help='import dns in bind format') + parser_describe.add_argument('zone', type=Zone, help='zone name') + parser_describe.add_argument('-f', '--file', type=argparse.FileType('r'), help='bind file') + parser_describe.set_defaults(func=cmd_import) + + parser_create = subparsers.add_parser('create', help='create a hosted zone') + parser_create.add_argument('zone', help='zone name') + parser_create.set_defaults(func=cmd_create) + + parser_delete = subparsers.add_parser('delete', help='delete a hosted zone') + parser_delete.add_argument('zone', type=Zone, help='zone name') + parser_delete.set_defaults(func=cmd_delete) + + parser_rrcreate = subparsers.add_parser('rrcreate', help='create a resource record') + parser_rrcreate.add_argument('zone', type=Zone, help='zone name') + parser_rrcreate.add_argument('rr', help='resource record') + parser_rrcreate.add_argument('type', choices=supported_rtypes, help='resource record type') + parser_rrcreate.add_argument('values', nargs='+', help='resource record values') + parser_rrcreate.add_argument('-x', '--ttl', type=int, default=86400, help='resource record ttl') + parser_rrcreate.add_argument('-r', '--replace', action='store_true', help='replace any existing record') + parser_rrcreate.set_defaults(func=cmd_rrcreate) + + parser_rrdelete = subparsers.add_parser('rrdelete', help='delete a resource record') + parser_rrdelete.add_argument('zone', type=Zone, help='zone name') + parser_rrdelete.add_argument('rr', help='resource record') + parser_rrdelete.add_argument('type', nargs='?', choices=supported_rtypes, help='resource record type') + parser_rrdelete.set_defaults(func=cmd_rrdelete) + + args = parser.parse_args() + args.func(args) + +if __name__=='__main__': + main() From 81a32096561b503be2453cea90666a867ec4b4fe Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Mon, 13 Dec 2010 23:14:40 +0000 Subject: [PATCH 02/46] Add more record types --- cli53.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/cli53.py b/cli53.py index 9e304674..2448a2bc 100755 --- a/cli53.py +++ b/cli53.py @@ -22,7 +22,8 @@ try: import dns.zone, dns.rdataset, dns.node, dns.rdtypes, dns.rdataclass - import dns.rdtypes.ANY.CNAME, dns.rdtypes.ANY.SOA, dns.rdtypes.ANY.MX, dns.rdtypes.ANY.NS, dns.rdtypes.IN.A + import dns.rdtypes.ANY.CNAME, dns.rdtypes.ANY.SOA, dns.rdtypes.ANY.MX, dns.rdtypes.ANY.SPF + import dns.rdtypes.ANY.TXT, dns.rdtypes.ANY.NS, dns.rdtypes.ANY.PTR, dns.rdtypes.IN.A, dns.rdtypes.IN.AAAA, dns.rdtypes.IN.SRV except ImportError: print "Please install dnspython:" print "easy_install dnspython" @@ -137,6 +138,8 @@ def _create_rdataset(rtype, ttl, values): rdtype = None if rtype == 'A': rdtype = dns.rdtypes.IN.A.A(dns.rdataclass.IN, dns.rdatatype.A, value) + elif rtype == 'AAAA': + rdtype = dns.rdtypes.IN.AAAA.AAAA(dns.rdataclass.IN, dns.rdatatype.AAAA, value) elif rtype == 'CNAME': rdtype = dns.rdtypes.ANY.CNAME.CNAME(dns.rdataclass.ANY, dns.rdatatype.CNAME, dns.name.from_text(value)) @@ -155,7 +158,20 @@ def _create_rdataset(rtype, ttl, values): elif rtype == 'MX': pref, ex = value.split() pref = int(pref) - rdtype = dns.rdtypes.ANY.MX.MX(dns.rdataclass.ANY, dns.rdatatype.SOA, pref, dns.name.from_text(ex)) + rdtype = dns.rdtypes.ANY.MX.MX(dns.rdataclass.ANY, dns.rdatatype.MX, pref, dns.name.from_text(ex)) + elif rtype == 'PTR': + rdtype = dns.rdtypes.ANY.PTR.PTR(dns.rdataclass.ANY, dns.rdatatype.PTR, value) + elif rtype == 'SPF': + rdtype = dns.rdtypes.ANY.SPF.SPF(dns.rdataclass.ANY, dns.rdatatype.SPF, value) + elif rtype == 'SRV': + priority, weight, port, target = value.split() + priority = int(priority) + weight = int(weight) + port = int(port) + target = dns.name.from_text(target) + rdtype = dns.rdtypes.IN.SRV.SRV(dns.rdataclass.IN, dns.rdatatype.SRV, priority, weight, port, target) + elif rtype == 'TXT': + rdtype = dns.rdtypes.ANY.TXT.TXT(dns.rdataclass.ANY, dns.rdatatype.TXT, value) else: raise ValueError, 'record type %s not handled' % rtype rdataset.items.append(rdtype) @@ -270,7 +286,7 @@ def main(): parser = argparse.ArgumentParser(description='route53 command line tool') subparsers = parser.add_subparsers(help='sub-command help') - supported_rtypes = ('A', 'CNAME', 'SOA', 'NS', 'MX') + supported_rtypes = ('A', 'AAAA', 'CNAME', 'SOA', 'NS', 'MX', 'PTR', 'SPF', 'SRV', 'TXT') parser_list = subparsers.add_parser('list', help='list hosted zones') parser_list.set_defaults(func=cmd_list) From 5fba3e5f328cb37b478387b91eccead80a28fb89 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Mon, 13 Dec 2010 23:24:37 +0000 Subject: [PATCH 03/46] Fix relative CNAMEs --- cli53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli53.py b/cli53.py index 2448a2bc..a4e278e2 100755 --- a/cli53.py +++ b/cli53.py @@ -104,7 +104,7 @@ def _xml_changes(self, zone, creates=[], deletes=[]): rrs = et.SubElement(rrset, 'ResourceRecords') for rdtype in rdataset.items: rr = et.SubElement(rrs, 'ResourceRecord') - text_element(rr, 'Value', rdtype.to_text()) + text_element(rr, 'Value', rdtype.to_text(origin=zone.origin, relativize=False)) out = StringIO() et.ElementTree(root).write(out) From 88b701344d715e177872a2557e3c30e2b411c385 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Tue, 14 Dec 2010 11:22:36 +0000 Subject: [PATCH 04/46] Add rrpurge action, fixes to cname/txt/mx record import --- cli53.py | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/cli53.py b/cli53.py index a4e278e2..e78dac94 100755 --- a/cli53.py +++ b/cli53.py @@ -35,6 +35,16 @@ print 'export AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXX' sys.exit(-1) +# Custom MX class to prevent changing values +class MX(dns.rdtypes.mxbase.MXBase): + def to_text(self, **kw): + return '%d %s' % (self.preference, self.exchange) + +# Custom CNAME class to prevent changing values +class CNAME(dns.rdtypes.nsbase.NSBase): + def to_text(self, **kw): + return self.target + r53 = boto.route53.Route53Connection() def pprint(obj, findent='', indent=''): @@ -78,6 +88,14 @@ def create_all(self, zone, exclude=()): if dns.rdatatype.to_text(rdataset.rdtype) not in exclude: creates.append((name, rdataset)) return self._xml_changes(zone, creates=creates) + + def delete_all(self, zone, exclude=()): + deletes = [] + for name, node in zone.items(): + for rdataset in node.rdatasets: + if dns.rdatatype.to_text(rdataset.rdtype) not in exclude: + deletes.append((name, rdataset)) + return self._xml_changes(zone, deletes=deletes) def create_record(self, zone, name, rdataset): return self._xml_changes(zone, creates=[(name,rdataset)]) @@ -121,6 +139,9 @@ def convert(self, info, xml): for rrsets in tree.findall("{%s}ResourceRecordSets" % ns): for rrset in rrsets.findall("{%s}ResourceRecordSet" % ns): name = rrset.find('{%s}Name' % ns).text + if '\\052' in name: + # * char seems to confuse Amazon and is returned as \\052 + name = name.replace('\\052', '*') rtype = rrset.find('{%s}Type' % ns).text ttl = int(rrset.find('{%s}TTL' % ns).text) @@ -131,6 +152,7 @@ def convert(self, info, xml): return z +re_quoted = re.compile(r'^".*"$') def _create_rdataset(rtype, ttl, values): rdataset = dns.rdataset.Rdataset(dns.rdataclass.IN, dns.rdatatype.from_text(rtype)) rdataset.ttl = ttl @@ -141,8 +163,7 @@ def _create_rdataset(rtype, ttl, values): elif rtype == 'AAAA': rdtype = dns.rdtypes.IN.AAAA.AAAA(dns.rdataclass.IN, dns.rdatatype.AAAA, value) elif rtype == 'CNAME': - rdtype = dns.rdtypes.ANY.CNAME.CNAME(dns.rdataclass.ANY, - dns.rdatatype.CNAME, dns.name.from_text(value)) + rdtype = CNAME(dns.rdataclass.ANY, dns.rdatatype.CNAME, value) elif rtype == 'SOA': mname, rname, serial, refresh, retry, expire, minimum = value.split() mname = dns.name.from_text(mname) @@ -158,7 +179,7 @@ def _create_rdataset(rtype, ttl, values): elif rtype == 'MX': pref, ex = value.split() pref = int(pref) - rdtype = dns.rdtypes.ANY.MX.MX(dns.rdataclass.ANY, dns.rdatatype.MX, pref, dns.name.from_text(ex)) + rdtype = MX(dns.rdataclass.ANY, dns.rdatatype.MX, pref, ex) elif rtype == 'PTR': rdtype = dns.rdtypes.ANY.PTR.PTR(dns.rdataclass.ANY, dns.rdatatype.PTR, value) elif rtype == 'SPF': @@ -171,6 +192,8 @@ def _create_rdataset(rtype, ttl, values): target = dns.name.from_text(target) rdtype = dns.rdtypes.IN.SRV.SRV(dns.rdataclass.IN, dns.rdatatype.SRV, priority, weight, port, target) elif rtype == 'TXT': + if re_quoted.match(value): + value = value[1:-1] rdtype = dns.rdtypes.ANY.TXT.TXT(dns.rdataclass.ANY, dns.rdatatype.TXT, value) else: raise ValueError, 'record type %s not handled' % rtype @@ -281,6 +304,14 @@ def cmd_rrdelete(args): else: print 'Record not found: %s' % args.rr +def cmd_rrpurge(args): + zone = _get_records(args) + f = BindToR53Formatter() + xml = f.delete_all(zone, exclude=('SOA','NS')) + print xml + ret = r53.change_rrsets(args.zone, xml) + pprint(ret.ChangeResourceRecordSetsResponse) + def main(): connection = boto.route53.Route53Connection() parser = argparse.ArgumentParser(description='route53 command line tool') @@ -331,6 +362,11 @@ def main(): parser_rrdelete.add_argument('type', nargs='?', choices=supported_rtypes, help='resource record type') parser_rrdelete.set_defaults(func=cmd_rrdelete) + parser_rrpurge = subparsers.add_parser('rrpurge', help='purge all resource records') + parser_rrpurge.add_argument('zone', type=Zone, help='zone name') + parser_rrpurge.add_argument('--confirm', required=True, action='store_true', help='confirm you definitely want to do this!') + parser_rrpurge.set_defaults(func=cmd_rrpurge) + args = parser.parse_args() args.func(args) From ca8d551a498d0d4fe600f2e2d3f6255fdb1390db Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Tue, 14 Dec 2010 20:25:24 +0000 Subject: [PATCH 05/46] Fix for import/rrpurge --- cli53.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/cli53.py b/cli53.py index e78dac94..14b1d2cb 100755 --- a/cli53.py +++ b/cli53.py @@ -80,20 +80,24 @@ def text_element(parent, name, text): el = et.SubElement(parent, name) el.text = text +def is_root_soa_or_ns(name, rdataset): + rt = dns.rdatatype.to_text(rdataset.rdtype) + return (rt in ('SOA', 'NS') and name.to_text() == '@') + class BindToR53Formatter(object): - def create_all(self, zone, exclude=()): + def create_all(self, zone, exclude=None): creates = [] for name, node in zone.items(): for rdataset in node.rdatasets: - if dns.rdatatype.to_text(rdataset.rdtype) not in exclude: + if not exclude or not exclude(name, rdataset): creates.append((name, rdataset)) return self._xml_changes(zone, creates=creates) - def delete_all(self, zone, exclude=()): + def delete_all(self, zone, exclude=None): deletes = [] for name, node in zone.items(): for rdataset in node.rdatasets: - if dns.rdatatype.to_text(rdataset.rdtype) not in exclude: + if not exclude or not exclude(name, rdataset): deletes.append((name, rdataset)) return self._xml_changes(zone, deletes=deletes) @@ -221,7 +225,7 @@ def cmd_import(args): zone = dns.zone.from_text(text, origin=origin, check_origin=True) f = BindToR53Formatter() - xml = f.create_all(zone, exclude=('SOA','NS')) + xml = f.create_all(zone, exclude=is_root_soa_or_ns) ret = r53.change_rrsets(args.zone, xml) pprint(ret.ChangeResourceRecordSetsResponse) @@ -252,7 +256,10 @@ def cmd_create(args): def cmd_delete(args): ret = r53.delete_hosted_zone(args.zone) - pprint(ret.DeleteHostedZoneResponse) + if hasattr(ret, 'ErrorResponse'): + pprint(ret.ErrorResponse) + else: + pprint(ret.DeleteHostedZoneResponse) def cmd_rrcreate(args): zone = _get_records(args) @@ -307,7 +314,7 @@ def cmd_rrdelete(args): def cmd_rrpurge(args): zone = _get_records(args) f = BindToR53Formatter() - xml = f.delete_all(zone, exclude=('SOA','NS')) + xml = f.delete_all(zone, exclude=is_root_soa_or_ns) print xml ret = r53.change_rrsets(args.zone, xml) pprint(ret.ChangeResourceRecordSetsResponse) From 9b9b802c0a2d072454d8cbe1b754986681e37adc Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Tue, 14 Dec 2010 20:30:28 +0000 Subject: [PATCH 06/46] README tweaks --- README.markdown | 4 ++-- cli53.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index bdae03f3..fde4f348 100644 --- a/README.markdown +++ b/README.markdown @@ -50,9 +50,9 @@ Further documentation is available, e.g.: Installation ------------ -There is no need to install, but you will need python, the latest boto library (from git) and dnspython: +There is no need to install, but you will need python, the latest boto library (from my git branch) and dnspython: - $ git clone boto && cd boto && python setup.py install + $ git clone git://github.com/barnybug/boto && cd boto && python setup.py install $ easy_install dnspython You need to set your Amazon credentials in the environment as AWS_ACCESS_KEY_ID diff --git a/cli53.py b/cli53.py index 14b1d2cb..23d51884 100755 --- a/cli53.py +++ b/cli53.py @@ -8,6 +8,7 @@ from cStringIO import StringIO # needs patched boto from github: http://github.com/barnybug/boto +# git clone git://github.com/barnybug/boto try: import boto.route53, boto.jsonresponse except ImportError: From 2870481c75b2c8a210c379dccef10b41ccad5497 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Wed, 15 Dec 2010 20:51:29 +0000 Subject: [PATCH 07/46] Spurious print --- cli53.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cli53.py b/cli53.py index 23d51884..af0cd238 100755 --- a/cli53.py +++ b/cli53.py @@ -316,7 +316,6 @@ def cmd_rrpurge(args): zone = _get_records(args) f = BindToR53Formatter() xml = f.delete_all(zone, exclude=is_root_soa_or_ns) - print xml ret = r53.change_rrsets(args.zone, xml) pprint(ret.ChangeResourceRecordSetsResponse) From 077a74b9965abf8c1e486d01f3c1a82f41f02bfa Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Wed, 15 Dec 2010 20:55:53 +0000 Subject: [PATCH 08/46] update README --- README.markdown | 4 ++-- cli53.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.markdown b/README.markdown index fde4f348..fcfcbabd 100644 --- a/README.markdown +++ b/README.markdown @@ -50,9 +50,9 @@ Further documentation is available, e.g.: Installation ------------ -There is no need to install, but you will need python, the latest boto library (from my git branch) and dnspython: +There is no need to install, but you will need python, the latest boto library (from git) and dnspython: - $ git clone git://github.com/barnybug/boto && cd boto && python setup.py install + $ git clone git://github.com/boto/boto && cd boto && python setup.py install $ easy_install dnspython You need to set your Amazon credentials in the environment as AWS_ACCESS_KEY_ID diff --git a/cli53.py b/cli53.py index af0cd238..c2195b3c 100755 --- a/cli53.py +++ b/cli53.py @@ -7,8 +7,8 @@ import re from cStringIO import StringIO -# needs patched boto from github: http://github.com/barnybug/boto -# git clone git://github.com/barnybug/boto +# needs latest boto from github: http://github.com/boto/boto +# git clone git://github.com/boto/boto try: import boto.route53, boto.jsonresponse except ImportError: From 8a78a6ae6359df731113630c06eaa20e148d121b Mon Sep 17 00:00:00 2001 From: joe miller Date: Thu, 16 Dec 2010 14:32:02 -0700 Subject: [PATCH 09/46] first commit of new '--wait' option that will cause cli53 to wait for update operations to reach INSYNC status before exiting (default: disabled) --- cli53.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/cli53.py b/cli53.py index c2195b3c..e42791ff 100755 --- a/cli53.py +++ b/cli53.py @@ -6,6 +6,7 @@ import os, sys import re from cStringIO import StringIO +from time import sleep # needs latest boto from github: http://github.com/boto/boto # git clone git://github.com/boto/boto @@ -253,20 +254,52 @@ def cmd_export(args): def cmd_create(args): ret = r53.create_hosted_zone(args.zone) - pprint(ret.CreateHostedZoneResponse) + if args.wait: + wait_for_sync(ret) + else: + pprint(ret.CreateHostedZoneResponse) def cmd_delete(args): ret = r53.delete_hosted_zone(args.zone) if hasattr(ret, 'ErrorResponse'): pprint(ret.ErrorResponse) + elif args.wait: + wait_for_sync(ret) else: pprint(ret.DeleteHostedZoneResponse) - + +def find_key_nonrecursive(adict, key): + stack = [adict] + while stack: + d = stack.pop() + if key in d: + return d[key] + for k, v in d.iteritems(): + if isinstance(v, dict): + stack.append(v) + +def wait_for_sync(obj): + waiting = 1 + id = find_key_nonrecursive(obj, 'Id') + id = id.replace('/change/', '') + sys.stdout.write("Waiting for change to sync") + ret = '' + while waiting: + ret = r53.get_change(id) + status = find_key_nonrecursive(ret, 'Status') + if status == 'INSYNC': + waiting = 0 + else: + sys.stdout.write('.') + sys.stdout.flush() + sleep(1) + print "completed" + pprint(ret.GetChangeResponse) + def cmd_rrcreate(args): zone = _get_records(args) name = dns.name.from_text(args.rr, zone.origin) rdataset = _create_rdataset(args.type, args.ttl, args.values) - rdataset_old = None node = zone.get_node(args.rr) if node: @@ -280,8 +313,11 @@ def cmd_rrcreate(args): else: xml = BindToR53Formatter().create_record(zone, name, rdataset) ret = r53.change_rrsets(args.zone, xml) - print 'Success' - pprint(ret.ChangeResourceRecordSetsResponse) + if args.wait: + wait_for_sync(ret) + else: + print 'Success' + pprint(ret.ChangeResourceRecordSetsResponse) def cmd_rrdelete(args): zone = _get_records(args) @@ -307,8 +343,11 @@ def cmd_rrdelete(args): xml = BindToR53Formatter().delete_record(zone, name, rdataset) ret = r53.change_rrsets(args.zone, xml) - print 'Success' - pprint(ret.ChangeResourceRecordSetsResponse) + if args.wait: + wait_for_sync(ret) + else: + print 'Success' + pprint(ret.ChangeResourceRecordSetsResponse) else: print 'Record not found: %s' % args.rr @@ -317,13 +356,17 @@ def cmd_rrpurge(args): f = BindToR53Formatter() xml = f.delete_all(zone, exclude=is_root_soa_or_ns) ret = r53.change_rrsets(args.zone, xml) - pprint(ret.ChangeResourceRecordSetsResponse) + if args.wait: + wait_for_sync(ret) + else: + pprint(ret.ChangeResourceRecordSetsResponse) def main(): connection = boto.route53.Route53Connection() parser = argparse.ArgumentParser(description='route53 command line tool') subparsers = parser.add_subparsers(help='sub-command help') - + + supported_rtypes = ('A', 'AAAA', 'CNAME', 'SOA', 'NS', 'MX', 'PTR', 'SPF', 'SRV', 'TXT') parser_list = subparsers.add_parser('list', help='list hosted zones') @@ -344,14 +387,17 @@ def main(): parser_describe = subparsers.add_parser('import', help='import dns in bind format') parser_describe.add_argument('zone', type=Zone, help='zone name') parser_describe.add_argument('-f', '--file', type=argparse.FileType('r'), help='bind file') + parser_describe.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_describe.set_defaults(func=cmd_import) parser_create = subparsers.add_parser('create', help='create a hosted zone') parser_create.add_argument('zone', help='zone name') + parser_create.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_create.set_defaults(func=cmd_create) parser_delete = subparsers.add_parser('delete', help='delete a hosted zone') parser_delete.add_argument('zone', type=Zone, help='zone name') + parser_delete.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_delete.set_defaults(func=cmd_delete) parser_rrcreate = subparsers.add_parser('rrcreate', help='create a resource record') @@ -361,17 +407,20 @@ def main(): parser_rrcreate.add_argument('values', nargs='+', help='resource record values') parser_rrcreate.add_argument('-x', '--ttl', type=int, default=86400, help='resource record ttl') parser_rrcreate.add_argument('-r', '--replace', action='store_true', help='replace any existing record') + parser_rrcreate.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_rrcreate.set_defaults(func=cmd_rrcreate) parser_rrdelete = subparsers.add_parser('rrdelete', help='delete a resource record') parser_rrdelete.add_argument('zone', type=Zone, help='zone name') parser_rrdelete.add_argument('rr', help='resource record') parser_rrdelete.add_argument('type', nargs='?', choices=supported_rtypes, help='resource record type') + parser_rrdelete.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_rrdelete.set_defaults(func=cmd_rrdelete) parser_rrpurge = subparsers.add_parser('rrpurge', help='purge all resource records') parser_rrpurge.add_argument('zone', type=Zone, help='zone name') parser_rrpurge.add_argument('--confirm', required=True, action='store_true', help='confirm you definitely want to do this!') + parser_rrpurge.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_rrpurge.set_defaults(func=cmd_rrpurge) args = parser.parse_args() From c60efca13a3ca62d1b08a12fe845d482458c4ce7 Mon Sep 17 00:00:00 2001 From: joe miller Date: Thu, 16 Dec 2010 14:40:40 -0700 Subject: [PATCH 10/46] removed some useless formatting changes --- cli53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli53.py b/cli53.py index e42791ff..b74898d5 100755 --- a/cli53.py +++ b/cli53.py @@ -300,6 +300,7 @@ def cmd_rrcreate(args): zone = _get_records(args) name = dns.name.from_text(args.rr, zone.origin) rdataset = _create_rdataset(args.type, args.ttl, args.values) + rdataset_old = None node = zone.get_node(args.rr) if node: @@ -365,7 +366,6 @@ def main(): connection = boto.route53.Route53Connection() parser = argparse.ArgumentParser(description='route53 command line tool') subparsers = parser.add_subparsers(help='sub-command help') - supported_rtypes = ('A', 'AAAA', 'CNAME', 'SOA', 'NS', 'MX', 'PTR', 'SPF', 'SRV', 'TXT') From 5f9607efbe12471db4a286257d74aba79b88eac0 Mon Sep 17 00:00:00 2001 From: joe miller Date: Thu, 16 Dec 2010 19:52:30 -0700 Subject: [PATCH 11/46] forgot to add wait_for_sync() to cmd_import(). fixed. --- cli53.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cli53.py b/cli53.py index b74898d5..55ce1ee5 100755 --- a/cli53.py +++ b/cli53.py @@ -230,7 +230,10 @@ def cmd_import(args): xml = f.create_all(zone, exclude=is_root_soa_or_ns) ret = r53.change_rrsets(args.zone, xml) - pprint(ret.ChangeResourceRecordSetsResponse) + if args.wait: + wait_for_sync(ret) + else: + pprint(ret.ChangeResourceRecordSetsResponse) re_zone_id = re.compile('^[A-Z0-9]{14}$') def Zone(zone): From 5b3e5354f30119808110ccc3ca1256ed214f3b7c Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Fri, 17 Dec 2010 09:38:07 +0000 Subject: [PATCH 12/46] Add --replace option for import and add $ORIGIN to export. Closes #1 --- cli53.py | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/cli53.py b/cli53.py index 55ce1ee5..1156b42c 100755 --- a/cli53.py +++ b/cli53.py @@ -87,21 +87,23 @@ def is_root_soa_or_ns(name, rdataset): return (rt in ('SOA', 'NS') and name.to_text() == '@') class BindToR53Formatter(object): - def create_all(self, zone, exclude=None): - creates = [] + def _build_list(self, zone, exclude=None): + li = [] for name, node in zone.items(): for rdataset in node.rdatasets: if not exclude or not exclude(name, rdataset): - creates.append((name, rdataset)) - return self._xml_changes(zone, creates=creates) + li.append((name, rdataset)) + return li + + def create_all(self, zone, old_zone=None, exclude=None): + creates = self._build_list(zone, exclude) + deletes = [] + if old_zone: + deletes = self._build_list(old_zone, exclude) + return self._xml_changes(zone, creates=creates, deletes=deletes) def delete_all(self, zone, exclude=None): - deletes = [] - for name, node in zone.items(): - for rdataset in node.rdatasets: - if not exclude or not exclude(name, rdataset): - deletes.append((name, rdataset)) - return self._xml_changes(zone, deletes=deletes) + return self._xml_changes(zone, deletes=self._build_list(zone, exclude)) def create_record(self, zone, name, rdataset): return self._xml_changes(zone, creates=[(name,rdataset)]) @@ -210,7 +212,6 @@ def cmd_xml(args): xml = r53.get_all_rrsets(args.zone) print xml -re_comment = re.compile('\S*;.*$') re_dos = re.compile('\r\n$') re_origin = re.compile(r'\$ORIGIN (\S+)') def cmd_import(args): @@ -226,9 +227,12 @@ def cmd_import(args): origin = m.group(1) zone = dns.zone.from_text(text, origin=origin, check_origin=True) + old_zone = None + if args.replace: + old_zone = _get_records(args) + f = BindToR53Formatter() - xml = f.create_all(zone, exclude=is_root_soa_or_ns) - + xml = f.create_all(zone, old_zone=old_zone, exclude=is_root_soa_or_ns) ret = r53.change_rrsets(args.zone, xml) if args.wait: wait_for_sync(ret) @@ -253,6 +257,7 @@ def _get_records(args): def cmd_export(args): zone = _get_records(args) + print '$ORIGIN %s' % zone.origin.to_text() zone.to_file(sys.stdout) def cmd_create(args): @@ -387,11 +392,12 @@ def main(): parser_describe.add_argument('zone', type=Zone, help='zone name') parser_describe.set_defaults(func=cmd_export) - parser_describe = subparsers.add_parser('import', help='import dns in bind format') - parser_describe.add_argument('zone', type=Zone, help='zone name') - parser_describe.add_argument('-f', '--file', type=argparse.FileType('r'), help='bind file') - parser_describe.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') - parser_describe.set_defaults(func=cmd_import) + parser_import = subparsers.add_parser('import', help='import dns in bind format') + parser_import.add_argument('zone', type=Zone, help='zone name') + parser_import.add_argument('-r', '--replace', action='store_true', help='replace all existing records (use with care!)') + parser_import.add_argument('-f', '--file', type=argparse.FileType('r'), help='bind file') + parser_import.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') + parser_import.set_defaults(func=cmd_import) parser_create = subparsers.add_parser('create', help='create a hosted zone') parser_create.add_argument('zone', help='zone name') From e33167016037c7aa263fffaa1820f7443c5690a9 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Fri, 17 Dec 2010 10:17:05 +0000 Subject: [PATCH 13/46] Improve error messages for zone lookup --- cli53.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cli53.py b/cli53.py index 1156b42c..8212f796 100755 --- a/cli53.py +++ b/cli53.py @@ -18,7 +18,7 @@ sys.exit(-1) import argparse -from argparse import ArgumentError +from argparse import ArgumentTypeError from types import StringTypes import xml.etree.ElementTree as et @@ -244,10 +244,14 @@ def Zone(zone): if re_zone_id.match(zone): return zone ret = r53.get_all_hosted_zones() - for hz in ret.ListHostedZonesResponse.HostedZones: - if hz.Name == zone or hz.Name == zone+'.': - return hz.Id.replace('/hostedzone/', '') - raise ArgumentError, 'Zone %s not found' % zone + + hzs = [ hz.Id.replace('/hostedzone/', '') for hz in ret.ListHostedZonesResponse.HostedZones if hz.Name == zone or hz.Name == zone+'.' ] + if len(hzs) == 1: + return hzs[0] + elif len(hzs) > 1: + raise ArgumentTypeError, 'Zone %r is ambiguous (matches: %s), please specify ID' % (zone, ', '.join(hzs)) + else: + raise ArgumentTypeError, 'Zone %r not found' % zone def _get_records(args): info = r53.get_hosted_zone(args.zone) From a2b379ee9ef04a369af3f0591c9740342e843040 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Fri, 17 Dec 2010 10:25:09 +0000 Subject: [PATCH 14/46] Add export --full option --- cli53.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli53.py b/cli53.py index 8212f796..604716f6 100755 --- a/cli53.py +++ b/cli53.py @@ -262,7 +262,7 @@ def _get_records(args): def cmd_export(args): zone = _get_records(args) print '$ORIGIN %s' % zone.origin.to_text() - zone.to_file(sys.stdout) + zone.to_file(sys.stdout, relativize=not args.full) def cmd_create(args): ret = r53.create_hosted_zone(args.zone) @@ -394,6 +394,7 @@ def main(): parser_describe = subparsers.add_parser('export', help='export dns in bind format') parser_describe.add_argument('zone', type=Zone, help='zone name') + parser_describe.add_argument('--full', action='store_true', help='export prefixes as full names') parser_describe.set_defaults(func=cmd_export) parser_import = subparsers.add_parser('import', help='import dns in bind format') From 07d8983355667a3e9b6dd112ebda5b839bb9620e Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Sun, 19 Dec 2010 17:33:40 +0000 Subject: [PATCH 15/46] Remove assumption on zone id length. Closes #3 --- cli53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli53.py b/cli53.py index 604716f6..027e8fc1 100755 --- a/cli53.py +++ b/cli53.py @@ -239,7 +239,7 @@ def cmd_import(args): else: pprint(ret.ChangeResourceRecordSetsResponse) -re_zone_id = re.compile('^[A-Z0-9]{14}$') +re_zone_id = re.compile('^[A-Z0-9]+$') def Zone(zone): if re_zone_id.match(zone): return zone From e825ff16fd941f050847de9902d16a624d846d5b Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Sun, 19 Dec 2010 21:04:00 +0000 Subject: [PATCH 16/46] Workaround dnspython for PTR, SRV and NS. Closes #4 --- cli53.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/cli53.py b/cli53.py index 027e8fc1..c2ddc980 100755 --- a/cli53.py +++ b/cli53.py @@ -24,8 +24,9 @@ try: import dns.zone, dns.rdataset, dns.node, dns.rdtypes, dns.rdataclass - import dns.rdtypes.ANY.CNAME, dns.rdtypes.ANY.SOA, dns.rdtypes.ANY.MX, dns.rdtypes.ANY.SPF - import dns.rdtypes.ANY.TXT, dns.rdtypes.ANY.NS, dns.rdtypes.ANY.PTR, dns.rdtypes.IN.A, dns.rdtypes.IN.AAAA, dns.rdtypes.IN.SRV + import dns.rdtypes.ANY.SOA, dns.rdtypes.ANY.SPF + import dns.rdtypes.ANY.TXT, dns.rdtypes.IN.A, dns.rdtypes.IN.AAAA + import dns.rdtypes.mxbase, dns.rdtypes.nsbase except ImportError: print "Please install dnspython:" print "easy_install dnspython" @@ -37,16 +38,26 @@ print 'export AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXX' sys.exit(-1) + # Custom MX class to prevent changing values class MX(dns.rdtypes.mxbase.MXBase): def to_text(self, **kw): return '%d %s' % (self.preference, self.exchange) -# Custom CNAME class to prevent changing values -class CNAME(dns.rdtypes.nsbase.NSBase): +# Custom base class to prevent changing values +class CustomBase(dns.rdtypes.nsbase.NSBase): def to_text(self, **kw): return self.target +class CNAME(CustomBase): + pass +class NS(CustomBase): + pass +class SRV(CustomBase): + pass +class PTR(CustomBase): + pass + r53 = boto.route53.Route53Connection() def pprint(obj, findent='', indent=''): @@ -183,22 +194,17 @@ def _create_rdataset(rtype, ttl, values): minimum = int(minimum) rdtype = dns.rdtypes.ANY.SOA.SOA(dns.rdataclass.ANY, dns.rdatatype.SOA, mname, rname, serial, refresh, retry, expire, minimum) elif rtype == 'NS': - rdtype = dns.rdtypes.ANY.NS.NS(dns.rdataclass.ANY, dns.rdatatype.SOA, dns.name.from_text(value)) + rdtype = NS(dns.rdataclass.ANY, dns.rdatatype.NS, value) elif rtype == 'MX': pref, ex = value.split() pref = int(pref) rdtype = MX(dns.rdataclass.ANY, dns.rdatatype.MX, pref, ex) elif rtype == 'PTR': - rdtype = dns.rdtypes.ANY.PTR.PTR(dns.rdataclass.ANY, dns.rdatatype.PTR, value) + rdtype = PTR(dns.rdataclass.ANY, dns.rdatatype.PTR, value) elif rtype == 'SPF': rdtype = dns.rdtypes.ANY.SPF.SPF(dns.rdataclass.ANY, dns.rdatatype.SPF, value) elif rtype == 'SRV': - priority, weight, port, target = value.split() - priority = int(priority) - weight = int(weight) - port = int(port) - target = dns.name.from_text(target) - rdtype = dns.rdtypes.IN.SRV.SRV(dns.rdataclass.IN, dns.rdatatype.SRV, priority, weight, port, target) + rdtype = SRV(dns.rdataclass.IN, dns.rdatatype.SRV, value) elif rtype == 'TXT': if re_quoted.match(value): value = value[1:-1] @@ -340,7 +346,7 @@ def cmd_rrdelete(args): if node: if len(node.rdatasets) > 1 and not args.type: rtypes = [ dns.rdatatype.to_text(rds.rdtype) for rds in node.rdatasets ] - print 'Ambigious record - several resource types for record %s found: %s' % (args.rr, ', '.join(rtypes)) + print 'Ambigious record - several resource types for record %r found: %s' % (args.rr, ', '.join(rtypes)) else: rdataset = None for rds in node.rdatasets: From 417fdfd1645529932bff70e562635f8bf019df0b Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Mon, 20 Dec 2010 21:20:15 +0000 Subject: [PATCH 17/46] Fix some tab indents --- cli53.py | 60 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/cli53.py b/cli53.py index c2ddc980..3597c2a7 100755 --- a/cli53.py +++ b/cli53.py @@ -273,7 +273,7 @@ def cmd_export(args): def cmd_create(args): ret = r53.create_hosted_zone(args.zone) if args.wait: - wait_for_sync(ret) + wait_for_sync(ret) else: pprint(ret.CreateHostedZoneResponse) @@ -287,33 +287,33 @@ def cmd_delete(args): pprint(ret.DeleteHostedZoneResponse) def find_key_nonrecursive(adict, key): - stack = [adict] - while stack: - d = stack.pop() - if key in d: - return d[key] - for k, v in d.iteritems(): - if isinstance(v, dict): - stack.append(v) - + stack = [adict] + while stack: + d = stack.pop() + if key in d: + return d[key] + for k, v in d.iteritems(): + if isinstance(v, dict): + stack.append(v) + def wait_for_sync(obj): - waiting = 1 - id = find_key_nonrecursive(obj, 'Id') - id = id.replace('/change/', '') - sys.stdout.write("Waiting for change to sync") - ret = '' - while waiting: - ret = r53.get_change(id) - status = find_key_nonrecursive(ret, 'Status') - if status == 'INSYNC': - waiting = 0 - else: - sys.stdout.write('.') - sys.stdout.flush() - sleep(1) - print "completed" - pprint(ret.GetChangeResponse) - + waiting = 1 + id = find_key_nonrecursive(obj, 'Id') + id = id.replace('/change/', '') + sys.stdout.write("Waiting for change to sync") + ret = '' + while waiting: + ret = r53.get_change(id) + status = find_key_nonrecursive(ret, 'Status') + if status == 'INSYNC': + waiting = 0 + else: + sys.stdout.write('.') + sys.stdout.flush() + sleep(1) + print "completed" + pprint(ret.GetChangeResponse) + def cmd_rrcreate(args): zone = _get_records(args) name = dns.name.from_text(args.rr, zone.origin) @@ -363,7 +363,7 @@ def cmd_rrdelete(args): xml = BindToR53Formatter().delete_record(zone, name, rdataset) ret = r53.change_rrsets(args.zone, xml) if args.wait: - wait_for_sync(ret) + wait_for_sync(ret) else: print 'Success' pprint(ret.ChangeResourceRecordSetsResponse) @@ -376,7 +376,7 @@ def cmd_rrpurge(args): xml = f.delete_all(zone, exclude=is_root_soa_or_ns) ret = r53.change_rrsets(args.zone, xml) if args.wait: - wait_for_sync(ret) + wait_for_sync(ret) else: pprint(ret.ChangeResourceRecordSetsResponse) @@ -384,7 +384,7 @@ def main(): connection = boto.route53.Route53Connection() parser = argparse.ArgumentParser(description='route53 command line tool') subparsers = parser.add_subparsers(help='sub-command help') - + supported_rtypes = ('A', 'AAAA', 'CNAME', 'SOA', 'NS', 'MX', 'PTR', 'SPF', 'SRV', 'TXT') parser_list = subparsers.add_parser('list', help='list hosted zones') From f2bbc3c7dc2f5cb4fa7e1020b11e23ec0830d64f Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Mon, 20 Dec 2010 22:52:22 +0000 Subject: [PATCH 18/46] Handling for >100 result sets in a zone --- cli53.py | 125 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 79 insertions(+), 46 deletions(-) diff --git a/cli53.py b/cli53.py index 3597c2a7..891eca31 100755 --- a/cli53.py +++ b/cli53.py @@ -5,6 +5,7 @@ import os, sys import re +import itertools from cStringIO import StringIO from time import sleep @@ -96,6 +97,13 @@ def text_element(parent, name, text): def is_root_soa_or_ns(name, rdataset): rt = dns.rdatatype.to_text(rdataset.rdtype) return (rt in ('SOA', 'NS') and name.to_text() == '@') + +def paginate(iterable, size): + it = iter(iterable) + item = list(itertools.islice(it, size)) + while item: + yield item + item = list(itertools.islice(it, size)) class BindToR53Formatter(object): def _build_list(self, zone, exclude=None): @@ -126,35 +134,58 @@ def replace_record(self, zone, name, rdataset_old, rdataset_new): return self._xml_changes(zone, creates=[(name,rdataset_new)], deletes=[(name,rdataset_old)]) def _xml_changes(self, zone, creates=[], deletes=[]): + for page in paginate(self._iter_changes(creates, deletes), 100): + yield self._batch_change(zone, page) + + def _iter_changes(self, creates, deletes): + for chg, rdatasets in (('DELETE', deletes), ('CREATE', creates)): + for name, rdataset in rdatasets: + yield chg, name, rdataset + + def _batch_change(self, zone, chgs): root = et.Element('ChangeResourceRecordSetsRequest', xmlns=boto.route53.Route53Connection.XMLNameSpace) change_batch = et.SubElement(root, 'ChangeBatch') changes = et.SubElement(change_batch, 'Changes') - for chg, rdatasets in (('DELETE', deletes), ('CREATE', creates)): - for name, rdataset in rdatasets: - change = et.SubElement(changes, 'Change') - text_element(change, 'Action', chg) - rrset = et.SubElement(change, 'ResourceRecordSet') - text_element(rrset, 'Name', name.derelativize(zone.origin).to_text()) - text_element(rrset, 'Type', dns.rdatatype.to_text(rdataset.rdtype)) - text_element(rrset, 'TTL', str(rdataset.ttl)) - rrs = et.SubElement(rrset, 'ResourceRecords') - for rdtype in rdataset.items: - rr = et.SubElement(rrs, 'ResourceRecord') - text_element(rr, 'Value', rdtype.to_text(origin=zone.origin, relativize=False)) + for chg, name, rdataset in chgs: + change = et.SubElement(changes, 'Change') + text_element(change, 'Action', chg) + rrset = et.SubElement(change, 'ResourceRecordSet') + text_element(rrset, 'Name', name.derelativize(zone.origin).to_text()) + text_element(rrset, 'Type', dns.rdatatype.to_text(rdataset.rdtype)) + text_element(rrset, 'TTL', str(rdataset.ttl)) + rrs = et.SubElement(rrset, 'ResourceRecords') + for rdtype in rdataset.items: + rr = et.SubElement(rrs, 'ResourceRecord') + text_element(rr, 'Value', rdtype.to_text(origin=zone.origin, relativize=False)) out = StringIO() et.ElementTree(root).write(out) return out.getvalue() class R53ToBindFormatter(object): - def convert(self, info, xml): - origin = info.HostedZone.Name - z = dns.zone.Zone(dns.name.from_text(origin)) + def get_all_rrsets(self, r53, ghz, zone): + xml = r53.get_all_rrsets(zone) + ret, nextname, nexttype = self.convert(ghz, xml) + while nextname: + xml = r53.get_all_rrsets(zone, name=nextname, type=nexttype) + ret, nextname, nexttype = self.convert(ghz, xml, ret) + return ret + + def convert(self, info, xml, z=None): + if not z: + origin = info.HostedZone.Name + z = dns.zone.Zone(dns.name.from_text(origin)) ns = boto.route53.Route53Connection.XMLNameSpace tree = et.fromstring(xml) + nextname = None + nexttype = None + if tree.find('{%s}IsTruncated' % ns).text == 'true': + nextname = tree.find('{%s}NextRecordName' % ns).text + nexttype = tree.find('{%s}NextRecordType' % ns).text + for rrsets in tree.findall("{%s}ResourceRecordSets" % ns): for rrset in rrsets.findall("{%s}ResourceRecordSet" % ns): name = rrset.find('{%s}Name' % ns).text @@ -169,7 +200,7 @@ def convert(self, info, xml): node = z.get_node(name, create=True) node.replace_rdataset(rdataset) - return z + return z, nextname, nexttype re_quoted = re.compile(r'^".*"$') def _create_rdataset(rtype, ttl, values): @@ -238,12 +269,12 @@ def cmd_import(args): old_zone = _get_records(args) f = BindToR53Formatter() - xml = f.create_all(zone, old_zone=old_zone, exclude=is_root_soa_or_ns) - ret = r53.change_rrsets(args.zone, xml) - if args.wait: - wait_for_sync(ret) - else: - pprint(ret.ChangeResourceRecordSetsResponse) + for xml in f.create_all(zone, old_zone=old_zone, exclude=is_root_soa_or_ns): + ret = r53.change_rrsets(args.zone, xml) + if args.wait: + wait_for_sync(ret) + else: + pprint(ret.ChangeResourceRecordSetsResponse) re_zone_id = re.compile('^[A-Z0-9]+$') def Zone(zone): @@ -261,9 +292,8 @@ def Zone(zone): def _get_records(args): info = r53.get_hosted_zone(args.zone) - xml = r53.get_all_rrsets(args.zone) f = R53ToBindFormatter() - return f.convert(info.GetHostedZoneResponse, xml) + return f.get_all_rrsets(r53, info.GetHostedZoneResponse, args.zone) def cmd_export(args): zone = _get_records(args) @@ -327,16 +357,18 @@ def cmd_rrcreate(args): rdataset_old = rds break + f = BindToR53Formatter() if args.replace and rdataset_old: - xml = BindToR53Formatter().replace_record(zone, name, rdataset_old, rdataset) - else: - xml = BindToR53Formatter().create_record(zone, name, rdataset) - ret = r53.change_rrsets(args.zone, xml) - if args.wait: - wait_for_sync(ret) + parts = f.replace_record(zone, name, rdataset_old, rdataset) else: - print 'Success' - pprint(ret.ChangeResourceRecordSetsResponse) + parts = f.create_record(zone, name, rdataset) + for xml in parts: + ret = r53.change_rrsets(args.zone, xml) + if args.wait: + wait_for_sync(ret) + else: + print 'Success' + pprint(ret.ChangeResourceRecordSetsResponse) def cmd_rrdelete(args): zone = _get_records(args) @@ -359,26 +391,27 @@ def cmd_rrdelete(args): return print 'Deleting %s %s...' % (args.rr, dns.rdatatype.to_text(rds.rdtype)) - - xml = BindToR53Formatter().delete_record(zone, name, rdataset) - ret = r53.change_rrsets(args.zone, xml) - if args.wait: - wait_for_sync(ret) - else: - print 'Success' - pprint(ret.ChangeResourceRecordSetsResponse) + + f = BindToR53Formatter() + for xml in f.delete_record(zone, name, rdataset): + ret = r53.change_rrsets(args.zone, xml) + if args.wait: + wait_for_sync(ret) + else: + print 'Success' + pprint(ret.ChangeResourceRecordSetsResponse) else: print 'Record not found: %s' % args.rr def cmd_rrpurge(args): zone = _get_records(args) f = BindToR53Formatter() - xml = f.delete_all(zone, exclude=is_root_soa_or_ns) - ret = r53.change_rrsets(args.zone, xml) - if args.wait: - wait_for_sync(ret) - else: - pprint(ret.ChangeResourceRecordSetsResponse) + for xml in f.delete_all(zone, exclude=is_root_soa_or_ns): + ret = r53.change_rrsets(args.zone, xml) + if args.wait: + wait_for_sync(ret) + else: + pprint(ret.ChangeResourceRecordSetsResponse) def main(): connection = boto.route53.Route53Connection() From 1790e1a8b899b1937f7e34f8b6693731f312152b Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Mon, 20 Dec 2010 22:58:48 +0000 Subject: [PATCH 19/46] Update README --- README.markdown | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.markdown b/README.markdown index fcfcbabd..ca5bf640 100644 --- a/README.markdown +++ b/README.markdown @@ -7,6 +7,7 @@ The latest Amazon service Route 53 is a great addition, but only has a rudimenta available at the time of release. This script fills that gap until things have caught up. Features: + - create hosted zones - delete hosted zones @@ -22,7 +23,7 @@ Features: - delete resource records - works with BIND format zone files we all know and love - no need to edit - XML! + <ChangeResourceRecordSetsRequest> XML! Getting Started --------------- @@ -36,6 +37,9 @@ Check what we've done: Import a BIND zone file: ./cli53.py import example.com --file zonefile.txt +Replace with an imported zone, waiting for completion: + ./cli53.py import example.com --file zonefile.txt --replace --wait + Manually create some records: ./cli53.py rrcreate example.com www A 192.168.0.1 --ttl 3600 ./cli53.py rrcreate example.com www A 192.168.0.2 --ttl 3600 --replace @@ -57,3 +61,11 @@ There is no need to install, but you will need python, the latest boto library ( You need to set your Amazon credentials in the environment as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. + +Caveats +------- +As Amazon limits operations to a maximum of 100 changes, if you +perform a large operation that changes over 100 resource records it +will be split. An operation that involves deletes, followed by updates +such as an import with --replace will very briefly leave the domain +inconsistent. You have been warned! From df44366e366ca4ce1b371c3278412843949cd729 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Mon, 10 Jan 2011 10:05:02 +0000 Subject: [PATCH 20/46] Better error message for CNAME --- cli53.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cli53.py b/cli53.py index 891eca31..04a3acb1 100755 --- a/cli53.py +++ b/cli53.py @@ -227,8 +227,11 @@ def _create_rdataset(rtype, ttl, values): elif rtype == 'NS': rdtype = NS(dns.rdataclass.ANY, dns.rdatatype.NS, value) elif rtype == 'MX': - pref, ex = value.split() - pref = int(pref) + try: + pref, ex = value.split() + pref = int(pref) + except ValueError: + raise ValueError, 'mx records required two parts: priority name' rdtype = MX(dns.rdataclass.ANY, dns.rdatatype.MX, pref, ex) elif rtype == 'PTR': rdtype = PTR(dns.rdataclass.ANY, dns.rdatatype.PTR, value) From af4c234d36d9ee4350b05f23427faf9ced498cd6 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Mon, 10 Jan 2011 15:01:28 +0000 Subject: [PATCH 21/46] Support for CentOS --- README.markdown | 15 +++++++++++++-- cli53.py | 5 ++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/README.markdown b/README.markdown index ca5bf640..002889fe 100644 --- a/README.markdown +++ b/README.markdown @@ -56,8 +56,19 @@ Installation ------------ There is no need to install, but you will need python, the latest boto library (from git) and dnspython: - $ git clone git://github.com/boto/boto && cd boto && python setup.py install - $ easy_install dnspython +### Ubuntu + + $ git clone git://github.com/boto/boto && cd boto && sudo python setup.py install + $ sudo easy_install dnspython + +### CentOS 5.x + +There are a couple of extra requirements as CentOS has the older python 2.4: + + $ git clone git://github.com/boto/boto && cd boto && sudo python setup.py install + $ sudo yum install python-elementtree + $ sudo easy_install uuid + $ sudo easy_install dnspython You need to set your Amazon credentials in the environment as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. diff --git a/cli53.py b/cli53.py index 04a3acb1..5649f45c 100755 --- a/cli53.py +++ b/cli53.py @@ -21,7 +21,10 @@ import argparse from argparse import ArgumentTypeError from types import StringTypes -import xml.etree.ElementTree as et +try: + import xml.etree.ElementTree as et +except ImportError: + import elementtree.ElementTree as et try: import dns.zone, dns.rdataset, dns.node, dns.rdtypes, dns.rdataclass From ffe275b58f9ac5fa0e802cbf54e5c85b2304e5f0 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Wed, 2 Feb 2011 13:35:59 +0000 Subject: [PATCH 22/46] Fix some interface changes in boto. Pagination and xml dump (does anyone use this?) needs following up on.. --- cli53.py | 55 +++++++++++++++++++++++++------------------------------ 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/cli53.py b/cli53.py index 5649f45c..7cdcbfd5 100755 --- a/cli53.py +++ b/cli53.py @@ -168,42 +168,38 @@ def _batch_change(self, zone, chgs): class R53ToBindFormatter(object): def get_all_rrsets(self, r53, ghz, zone): - xml = r53.get_all_rrsets(zone) - ret, nextname, nexttype = self.convert(ghz, xml) + rrsets = r53.get_all_rrsets(zone) + ret, nextname, nexttype = self.convert(ghz, rrsets) while nextname: - xml = r53.get_all_rrsets(zone, name=nextname, type=nexttype) - ret, nextname, nexttype = self.convert(ghz, xml, ret) + rrsets = r53.get_all_rrsets(zone, name=nextname, type=nexttype) + ret, nextname, nexttype = self.convert(ghz, rrsets, ret) return ret - def convert(self, info, xml, z=None): + def convert(self, info, rrsets, z=None): if not z: origin = info.HostedZone.Name z = dns.zone.Zone(dns.name.from_text(origin)) - ns = boto.route53.Route53Connection.XMLNameSpace - tree = et.fromstring(xml) + # boto has lost this information in an interface change :-( +# nextname = None +# nexttype = None +# if tree.find('{%s}IsTruncated' % ns).text == 'true': +# nextname = tree.find('{%s}NextRecordName' % ns).text +# nexttype = tree.find('{%s}NextRecordType' % ns).text - nextname = None - nexttype = None - if tree.find('{%s}IsTruncated' % ns).text == 'true': - nextname = tree.find('{%s}NextRecordName' % ns).text - nexttype = tree.find('{%s}NextRecordType' % ns).text - - for rrsets in tree.findall("{%s}ResourceRecordSets" % ns): - for rrset in rrsets.findall("{%s}ResourceRecordSet" % ns): - name = rrset.find('{%s}Name' % ns).text - if '\\052' in name: - # * char seems to confuse Amazon and is returned as \\052 - name = name.replace('\\052', '*') - rtype = rrset.find('{%s}Type' % ns).text - ttl = int(rrset.find('{%s}TTL' % ns).text) - - values = [ rr.text for rr in rrset.findall('{%(ns)s}ResourceRecords/{%(ns)s}ResourceRecord/{%(ns)s}Value' % {'ns':ns}) ] - rdataset = _create_rdataset(rtype, ttl, values) - node = z.get_node(name, create=True) - node.replace_rdataset(rdataset) + for rrset in rrsets: + name = rrset.name + if '\\052' in name: + # * char seems to confuse Amazon and is returned as \\052 + name = name.replace('\\052', '*') + rtype = rrset.type + ttl = int(rrset.ttl) + + rdataset = _create_rdataset(rtype, ttl, rrset.resource_records) + node = z.get_node(name, create=True) + node.replace_rdataset(rdataset) - return z, nextname, nexttype + return z, None, None re_quoted = re.compile(r'^".*"$') def _create_rdataset(rtype, ttl, values): @@ -245,15 +241,14 @@ def _create_rdataset(rtype, ttl, values): elif rtype == 'TXT': if re_quoted.match(value): value = value[1:-1] - rdtype = dns.rdtypes.ANY.TXT.TXT(dns.rdataclass.ANY, dns.rdatatype.TXT, value) + rdtype = dns.rdtypes.ANY.TXT.TXT(dns.rdataclass.ANY, dns.rdatatype.TXT, [value]) else: raise ValueError, 'record type %s not handled' % rtype rdataset.items.append(rdtype) return rdataset def cmd_xml(args): - xml = r53.get_all_rrsets(args.zone) - print xml + print 'This functionality is no longer available due to changes in the boto library.' re_dos = re.compile('\r\n$') re_origin = re.compile(r'\$ORIGIN (\S+)') From a605c716c990ac36aa0073e5a56de5f0ce259c55 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Wed, 2 Feb 2011 16:50:49 +0000 Subject: [PATCH 23/46] Code tidy --- cli53.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/cli53.py b/cli53.py index 7cdcbfd5..f251aefa 100755 --- a/cli53.py +++ b/cli53.py @@ -168,25 +168,14 @@ def _batch_change(self, zone, chgs): class R53ToBindFormatter(object): def get_all_rrsets(self, r53, ghz, zone): - rrsets = r53.get_all_rrsets(zone) - ret, nextname, nexttype = self.convert(ghz, rrsets) - while nextname: - rrsets = r53.get_all_rrsets(zone, name=nextname, type=nexttype) - ret, nextname, nexttype = self.convert(ghz, rrsets, ret) - return ret + rrsets = r53.get_all_rrsets(zone, maxitems=10) + return self.convert(ghz, rrsets) def convert(self, info, rrsets, z=None): if not z: origin = info.HostedZone.Name z = dns.zone.Zone(dns.name.from_text(origin)) - # boto has lost this information in an interface change :-( -# nextname = None -# nexttype = None -# if tree.find('{%s}IsTruncated' % ns).text == 'true': -# nextname = tree.find('{%s}NextRecordName' % ns).text -# nexttype = tree.find('{%s}NextRecordType' % ns).text - for rrset in rrsets: name = rrset.name if '\\052' in name: @@ -199,7 +188,7 @@ def convert(self, info, rrsets, z=None): node = z.get_node(name, create=True) node.replace_rdataset(rdataset) - return z, None, None + return z re_quoted = re.compile(r'^".*"$') def _create_rdataset(rtype, ttl, values): From 3488841a31901d0577772a12f6cd036e7e98aa24 Mon Sep 17 00:00:00 2001 From: Jo Liss Date: Fri, 4 Feb 2011 20:19:57 +0100 Subject: [PATCH 24/46] Remove xml command from cli53.py --help --- cli53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli53.py b/cli53.py index f251aefa..d957c880 100755 --- a/cli53.py +++ b/cli53.py @@ -417,7 +417,7 @@ def main(): parser_list.add_argument('zone', type=Zone, help='zone name') parser_list.set_defaults(func=cmd_info) - parser_describe = subparsers.add_parser('xml', help='get the rrsets xml of a hosted zone') + parser_describe = subparsers.add_parser('xml') parser_describe.add_argument('zone', type=Zone, help='zone name') parser_describe.set_defaults(func=cmd_xml) From f5c102acab21998ba4d5e9581ca6150f5ade5972 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Tue, 8 Feb 2011 09:40:19 +0000 Subject: [PATCH 25/46] Fix SPF records --- cli53.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli53.py b/cli53.py index d957c880..e95ecf89 100755 --- a/cli53.py +++ b/cli53.py @@ -61,6 +61,8 @@ class SRV(CustomBase): pass class PTR(CustomBase): pass +class SPF(CustomBase): + pass r53 = boto.route53.Route53Connection() @@ -224,7 +226,7 @@ def _create_rdataset(rtype, ttl, values): elif rtype == 'PTR': rdtype = PTR(dns.rdataclass.ANY, dns.rdatatype.PTR, value) elif rtype == 'SPF': - rdtype = dns.rdtypes.ANY.SPF.SPF(dns.rdataclass.ANY, dns.rdatatype.SPF, value) + rdtype = SPF(dns.rdataclass.ANY, dns.rdatatype.SPF, value) elif rtype == 'SRV': rdtype = SRV(dns.rdataclass.IN, dns.rdatatype.SRV, value) elif rtype == 'TXT': From c010475e9ac30daef6ed4d6fa3db2dd3467183cd Mon Sep 17 00:00:00 2001 From: Mike Krieger Date: Fri, 4 Mar 2011 15:12:55 -0800 Subject: [PATCH 26/46] Adding support for listing resource records (rrlist) --- cli53.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cli53.py b/cli53.py index e95ecf89..506e01f7 100755 --- a/cli53.py +++ b/cli53.py @@ -405,6 +405,12 @@ def cmd_rrpurge(args): else: pprint(ret.ChangeResourceRecordSetsResponse) +def cmd_rrlist(args): + zone = _get_records(args) + print '\t'.join(["host","ttl","cls","type","data"]) + for record_name, record_value in zone.iteritems(): + print '\t'.join(record_value.to_text(record_name).split(' ')) + def main(): connection = boto.route53.Route53Connection() parser = argparse.ArgumentParser(description='route53 command line tool') @@ -468,6 +474,10 @@ def main(): parser_rrpurge.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_rrpurge.set_defaults(func=cmd_rrpurge) + parser_rrlist = subparsers.add_parser('rrlist', help='list all resource records') + parser_rrlist.add_argument('zone', type=Zone, help='zone name') + parser_rrlist.set_defaults(func=cmd_rrlist) + args = parser.parse_args() args.func(args) From 6beec1be2e73ec9cf95f54505800d03d5d243917 Mon Sep 17 00:00:00 2001 From: Martin Cozzi Date: Wed, 11 May 2011 12:18:58 -0700 Subject: [PATCH 27/46] Added buildout to deal with dependencies. --- .gitignore | 10 ++ README.markdown | 19 +- bootstrap.py | 260 +++++++++++++++++++++++++++ buildout.cfg | 20 +++ setup.py | 25 +++ src/cli53/__init__.py | 0 argparse.py => src/cli53/argparse.py | 0 cli53.py => src/cli53/cli53.py | 0 8 files changed, 321 insertions(+), 13 deletions(-) create mode 100644 .gitignore create mode 100644 bootstrap.py create mode 100644 buildout.cfg create mode 100644 setup.py create mode 100644 src/cli53/__init__.py rename argparse.py => src/cli53/argparse.py (100%) rename cli53.py => src/cli53/cli53.py (100%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..f8cd47fc --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +bin/ +eggs/ +develop-eggs/ +parts/ +downloads/ +*.egg-info/ +*.db +*.pyc +.installed.cfg +*.swp diff --git a/README.markdown b/README.markdown index 002889fe..ee80abd0 100644 --- a/README.markdown +++ b/README.markdown @@ -54,21 +54,14 @@ Further documentation is available, e.g.: Installation ------------ -There is no need to install, but you will need python, the latest boto library (from git) and dnspython: -### Ubuntu +### Buildout Install +Buildout will download all the dependencies needed in order to run cli53. +$ python bootstrap.py +$ bin/buildout - $ git clone git://github.com/boto/boto && cd boto && sudo python setup.py install - $ sudo easy_install dnspython - -### CentOS 5.x - -There are a couple of extra requirements as CentOS has the older python 2.4: - - $ git clone git://github.com/boto/boto && cd boto && sudo python setup.py install - $ sudo yum install python-elementtree - $ sudo easy_install uuid - $ sudo easy_install dnspython +Once the dependencies downloaded, run cli53 +$ bin/cli53 You need to set your Amazon credentials in the environment as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. diff --git a/bootstrap.py b/bootstrap.py new file mode 100644 index 00000000..5f2cb083 --- /dev/null +++ b/bootstrap.py @@ -0,0 +1,260 @@ +############################################################################## +# +# Copyright (c) 2006 Zope Foundation and Contributors. +# All Rights Reserved. +# +# This software is subject to the provisions of the Zope Public License, +# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. +# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED +# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS +# FOR A PARTICULAR PURPOSE. +# +############################################################################## +"""Bootstrap a buildout-based project + +Simply run this script in a directory containing a buildout.cfg. +The script accepts buildout command-line options, so you can +use the -c option to specify an alternate configuration file. +""" + +import os, shutil, sys, tempfile, textwrap, urllib, urllib2, subprocess +from optparse import OptionParser + +if sys.platform == 'win32': + def quote(c): + if ' ' in c: + return '"%s"' % c # work around spawn lamosity on windows + else: + return c +else: + quote = str + +# See zc.buildout.easy_install._has_broken_dash_S for motivation and comments. +stdout, stderr = subprocess.Popen( + [sys.executable, '-Sc', + 'try:\n' + ' import ConfigParser\n' + 'except ImportError:\n' + ' print 1\n' + 'else:\n' + ' print 0\n'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() +has_broken_dash_S = bool(int(stdout.strip())) + +# In order to be more robust in the face of system Pythons, we want to +# run without site-packages loaded. This is somewhat tricky, in +# particular because Python 2.6's distutils imports site, so starting +# with the -S flag is not sufficient. However, we'll start with that: +if not has_broken_dash_S and 'site' in sys.modules: + # We will restart with python -S. + args = sys.argv[:] + args[0:0] = [sys.executable, '-S'] + args = map(quote, args) + os.execv(sys.executable, args) +# Now we are running with -S. We'll get the clean sys.path, import site +# because distutils will do it later, and then reset the path and clean +# out any namespace packages from site-packages that might have been +# loaded by .pth files. +clean_path = sys.path[:] +import site +sys.path[:] = clean_path +for k, v in sys.modules.items(): + if k in ('setuptools', 'pkg_resources') or ( + hasattr(v, '__path__') and + len(v.__path__)==1 and + not os.path.exists(os.path.join(v.__path__[0],'__init__.py'))): + # This is a namespace package. Remove it. + sys.modules.pop(k) + +is_jython = sys.platform.startswith('java') + +setuptools_source = 'http://peak.telecommunity.com/dist/ez_setup.py' +distribute_source = 'http://python-distribute.org/distribute_setup.py' + +# parsing arguments +def normalize_to_url(option, opt_str, value, parser): + if value: + if '://' not in value: # It doesn't smell like a URL. + value = 'file://%s' % ( + urllib.pathname2url( + os.path.abspath(os.path.expanduser(value))),) + if opt_str == '--download-base' and not value.endswith('/'): + # Download base needs a trailing slash to make the world happy. + value += '/' + else: + value = None + name = opt_str[2:].replace('-', '_') + setattr(parser.values, name, value) + +usage = '''\ +[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] + +Bootstraps a buildout-based project. + +Simply run this script in a directory containing a buildout.cfg, using the +Python that you want bin/buildout to use. + +Note that by using --setup-source and --download-base to point to +local resources, you can keep this script from going over the network. +''' + +parser = OptionParser(usage=usage) +parser.add_option("-v", "--version", dest="version", + help="use a specific zc.buildout version") +parser.add_option("-d", "--distribute", + action="store_true", dest="use_distribute", default=False, + help="Use Distribute rather than Setuptools.") +parser.add_option("--setup-source", action="callback", dest="setup_source", + callback=normalize_to_url, nargs=1, type="string", + help=("Specify a URL or file location for the setup file. " + "If you use Setuptools, this will default to " + + setuptools_source + "; if you use Distribute, this " + "will default to " + distribute_source +".")) +parser.add_option("--download-base", action="callback", dest="download_base", + callback=normalize_to_url, nargs=1, type="string", + help=("Specify a URL or directory for downloading " + "zc.buildout and either Setuptools or Distribute. " + "Defaults to PyPI.")) +parser.add_option("--eggs", + help=("Specify a directory for storing eggs. Defaults to " + "a temporary directory that is deleted when the " + "bootstrap script completes.")) +parser.add_option("-t", "--accept-buildout-test-releases", + dest='accept_buildout_test_releases', + action="store_true", default=False, + help=("Normally, if you do not specify a --version, the " + "bootstrap script and buildout gets the newest " + "*final* versions of zc.buildout and its recipes and " + "extensions for you. If you use this flag, " + "bootstrap and buildout will get the newest releases " + "even if they are alphas or betas.")) +parser.add_option("-c", None, action="store", dest="config_file", + help=("Specify the path to the buildout configuration " + "file to be used.")) + +options, args = parser.parse_args() + +# if -c was provided, we push it back into args for buildout's main function +if options.config_file is not None: + args += ['-c', options.config_file] + +if options.eggs: + eggs_dir = os.path.abspath(os.path.expanduser(options.eggs)) +else: + eggs_dir = tempfile.mkdtemp() + +if options.setup_source is None: + if options.use_distribute: + options.setup_source = distribute_source + else: + options.setup_source = setuptools_source + +if options.accept_buildout_test_releases: + args.append('buildout:accept-buildout-test-releases=true') +args.append('bootstrap') + +try: + import pkg_resources + import setuptools # A flag. Sometimes pkg_resources is installed alone. + if not hasattr(pkg_resources, '_distribute'): + raise ImportError +except ImportError: + ez_code = urllib2.urlopen( + options.setup_source).read().replace('\r\n', '\n') + ez = {} + exec ez_code in ez + setup_args = dict(to_dir=eggs_dir, download_delay=0) + if options.download_base: + setup_args['download_base'] = options.download_base + if options.use_distribute: + setup_args['no_fake'] = True + ez['use_setuptools'](**setup_args) + if 'pkg_resources' in sys.modules: + reload(sys.modules['pkg_resources']) + import pkg_resources + # This does not (always?) update the default working set. We will + # do it. + for path in sys.path: + if path not in pkg_resources.working_set.entries: + pkg_resources.working_set.add_entry(path) + +cmd = [quote(sys.executable), + '-c', + quote('from setuptools.command.easy_install import main; main()'), + '-mqNxd', + quote(eggs_dir)] + +if not has_broken_dash_S: + cmd.insert(1, '-S') + +find_links = options.download_base +if not find_links: + find_links = os.environ.get('bootstrap-testing-find-links') +if find_links: + cmd.extend(['-f', quote(find_links)]) + +if options.use_distribute: + setup_requirement = 'distribute' +else: + setup_requirement = 'setuptools' +ws = pkg_resources.working_set +setup_requirement_path = ws.find( + pkg_resources.Requirement.parse(setup_requirement)).location +env = dict( + os.environ, + PYTHONPATH=setup_requirement_path) + +requirement = 'zc.buildout' +version = options.version +if version is None and not options.accept_buildout_test_releases: + # Figure out the most recent final version of zc.buildout. + import setuptools.package_index + _final_parts = '*final-', '*final' + def _final_version(parsed_version): + for part in parsed_version: + if (part[:1] == '*') and (part not in _final_parts): + return False + return True + index = setuptools.package_index.PackageIndex( + search_path=[setup_requirement_path]) + if find_links: + index.add_find_links((find_links,)) + req = pkg_resources.Requirement.parse(requirement) + if index.obtain(req) is not None: + best = [] + bestv = None + for dist in index[req.project_name]: + distv = dist.parsed_version + if _final_version(distv): + if bestv is None or distv > bestv: + best = [dist] + bestv = distv + elif distv == bestv: + best.append(dist) + if best: + best.sort() + version = best[-1].version +if version: + requirement = '=='.join((requirement, version)) +cmd.append(requirement) + +if is_jython: + import subprocess + exitcode = subprocess.Popen(cmd, env=env).wait() +else: # Windows prefers this, apparently; otherwise we would prefer subprocess + exitcode = os.spawnle(*([os.P_WAIT, sys.executable] + cmd + [env])) +if exitcode != 0: + sys.stdout.flush() + sys.stderr.flush() + print ("An error occurred when trying to install zc.buildout. " + "Look above this message for any errors that " + "were output by easy_install.") + sys.exit(exitcode) + +ws.add_entry(eggs_dir) +ws.require(requirement) +import zc.buildout.buildout +zc.buildout.buildout.main(args) +if not options.eggs: # clean up temporary egg directory + shutil.rmtree(eggs_dir) diff --git a/buildout.cfg b/buildout.cfg new file mode 100644 index 00000000..50f7b1ab --- /dev/null +++ b/buildout.cfg @@ -0,0 +1,20 @@ +[buildout] +develop = . +parts = cli53 +relative-paths = true + +[cli53] +recipe = zc.recipe.egg +interpreter = python +eggs = cli53 + +[versions] +boto = 2.0b4 +dnspython = 1.9.4 +elementtree = 1.2.6-20050316 +uuid = 1.30 + + +[scripts] +recipe = zc.recipe.egg:scripts +eggs = cli53 diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..8cb2c19d --- /dev/null +++ b/setup.py @@ -0,0 +1,25 @@ +#!/usr/bin/python2.6 +"""Setup file for cli53.""" + +__author__ = 'contact@martincozzi.com' + +from setuptools import setup + +setup( + name='cli53', + version='0.1', + description='Command line script to administer the Amazon Route 53 dns service.', + package_dir={'': 'src'}, + install_requires=[ + 'boto', + 'dnspython', + 'elementtree', + 'uuid', + ], + entry_points={ + 'console_scripts': [ + 'cli53 = cli53.cli53:main', + ], + }, + zip_safe=False, + ) diff --git a/src/cli53/__init__.py b/src/cli53/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/argparse.py b/src/cli53/argparse.py similarity index 100% rename from argparse.py rename to src/cli53/argparse.py diff --git a/cli53.py b/src/cli53/cli53.py similarity index 100% rename from cli53.py rename to src/cli53/cli53.py From 1229310636f33f9a6d98b2afe650ebbff7693869 Mon Sep 17 00:00:00 2001 From: Martin Cozzi Date: Wed, 11 May 2011 12:21:03 -0700 Subject: [PATCH 28/46] Formatting README to look better. --- README.markdown | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.markdown b/README.markdown index ee80abd0..28fb8495 100644 --- a/README.markdown +++ b/README.markdown @@ -57,11 +57,11 @@ Installation ### Buildout Install Buildout will download all the dependencies needed in order to run cli53. -$ python bootstrap.py -$ bin/buildout + $ python bootstrap.py + $ bin/buildout Once the dependencies downloaded, run cli53 -$ bin/cli53 + $ bin/cli53 You need to set your Amazon credentials in the environment as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. From c780c14d5eab3ff1919b8c64ebe60df74e908f07 Mon Sep 17 00:00:00 2001 From: Martin Cozzi Date: Wed, 11 May 2011 12:22:53 -0700 Subject: [PATCH 29/46] Edited README.markdown via GitHub --- README.markdown | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.markdown b/README.markdown index 28fb8495..cfa2d249 100644 --- a/README.markdown +++ b/README.markdown @@ -57,11 +57,15 @@ Installation ### Buildout Install Buildout will download all the dependencies needed in order to run cli53. - $ python bootstrap.py - $ bin/buildout + + $ python bootstrap.py + $ bin/buildout + Once the dependencies downloaded, run cli53 - $ bin/cli53 + + $ bin/cli53 + You need to set your Amazon credentials in the environment as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. From 0d7c3856581fb35c2bfa26b25207a09abbd97c04 Mon Sep 17 00:00:00 2001 From: Martin Cozzi Date: Fri, 20 May 2011 16:44:49 -0700 Subject: [PATCH 30/46] Fix display on readme. Remove cli53.py with bin/cli53 --- README.markdown | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/README.markdown b/README.markdown index cfa2d249..5d0d5d05 100644 --- a/README.markdown +++ b/README.markdown @@ -29,28 +29,29 @@ Getting Started --------------- Create a hosted zone: - ./cli53.py create example.com + $ bin/cli53 create example.com Check what we've done: - ./cli53.py list + $ bin/cli53 list Import a BIND zone file: - ./cli53.py import example.com --file zonefile.txt + $ bin/cli53 import example.com --file zonefile.txt Replace with an imported zone, waiting for completion: - ./cli53.py import example.com --file zonefile.txt --replace --wait + $ bin/cli53 import example.com --file zonefile.txt --replace --wait Manually create some records: - ./cli53.py rrcreate example.com www A 192.168.0.1 --ttl 3600 - ./cli53.py rrcreate example.com www A 192.168.0.2 --ttl 3600 --replace - ./cli53.py rrcreate example.com '' MX '10 192.168.0.1' '20 192.168.0.2' + $ bin/cli53 rrcreate example.com www A 192.168.0.1 --ttl 3600 + $ bin/cli53 rrcreate example.com www A 192.168.0.2 --ttl 3600 --replace + $ bin/cli53 rrcreate example.com '' MX '10 192.168.0.1' '20 192.168.0.2' Export as a BIND zone file (useful for checking): - ./cli53.py export example.com + $ bin/cli53 export example.com Further documentation is available, e.g.: - ./cli53.py --help - ./cli53.py rrcreate --help + $ bin/cli53 --help + $ bin/cli53 rrcreate --help + Installation ------------ From 9f2efe06efca25747fbc33d1f3f07057065478dc Mon Sep 17 00:00:00 2001 From: Martin Cozzi Date: Fri, 20 May 2011 16:45:17 -0700 Subject: [PATCH 31/46] Edited README.markdown via GitHub --- README.markdown | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.markdown b/README.markdown index 5d0d5d05..de34c74a 100644 --- a/README.markdown +++ b/README.markdown @@ -29,26 +29,33 @@ Getting Started --------------- Create a hosted zone: + $ bin/cli53 create example.com Check what we've done: + $ bin/cli53 list Import a BIND zone file: + $ bin/cli53 import example.com --file zonefile.txt Replace with an imported zone, waiting for completion: + $ bin/cli53 import example.com --file zonefile.txt --replace --wait Manually create some records: + $ bin/cli53 rrcreate example.com www A 192.168.0.1 --ttl 3600 $ bin/cli53 rrcreate example.com www A 192.168.0.2 --ttl 3600 --replace $ bin/cli53 rrcreate example.com '' MX '10 192.168.0.1' '20 192.168.0.2' Export as a BIND zone file (useful for checking): + $ bin/cli53 export example.com Further documentation is available, e.g.: + $ bin/cli53 --help $ bin/cli53 rrcreate --help From 1156117fa5ed41a696c84d6d4fd1bf448d700005 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Thu, 21 Jul 2011 09:24:22 +0100 Subject: [PATCH 32/46] Updates to credentials. Closes #15 --- src/cli53/cli53.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index e95ecf89..4d34d3c8 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -12,7 +12,7 @@ # needs latest boto from github: http://github.com/boto/boto # git clone git://github.com/boto/boto try: - import boto.route53, boto.jsonresponse + import boto.route53, boto.jsonresponse, boto.exception except ImportError: print "Please install latest boto:" print "git clone boto && cd boto && python setup.py install" @@ -36,13 +36,6 @@ print "easy_install dnspython" sys.exit(-1) -if not (os.getenv('AWS_ACCESS_KEY_ID') and os.getenv('AWS_SECRET_ACCESS_KEY')): - print 'Please set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, e.g.:' - print 'export AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXX' - print 'export AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXX' - sys.exit(-1) - - # Custom MX class to prevent changing values class MX(dns.rdtypes.mxbase.MXBase): def to_text(self, **kw): @@ -64,7 +57,21 @@ class PTR(CustomBase): class SPF(CustomBase): pass -r53 = boto.route53.Route53Connection() +try: + r53 = boto.route53.Route53Connection() +except boto.exception.NoAuthHandlerFound: + print 'Please configure your AWS credentials, either through environment variables or' + print '~/.boto config file.' + print 'e.g.' + print 'export AWS_ACCESS_KEY_ID=XXXXXXXXXXXXXX' + print 'export AWS_SECRET_ACCESS_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + print 'or in ~/.boto:' + print '[Credentials]' + print 'aws_access_key_id = XXXXXXXXXXXXXX' + print 'aws_secret_access_key = XXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + print + print 'See: http://code.google.com/p/boto/wiki/BotoConfig' + sys.exit(-1) def pprint(obj, findent='', indent=''): if isinstance(obj, StringTypes): From ba001c9c967c5e905afed9147506fb24db1187f7 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Thu, 21 Jul 2011 09:27:54 +0100 Subject: [PATCH 33/46] README updates --- README.markdown | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.markdown b/README.markdown index cfa2d249..33d854ef 100644 --- a/README.markdown +++ b/README.markdown @@ -61,14 +61,20 @@ Buildout will download all the dependencies needed in order to run cli53. $ python bootstrap.py $ bin/buildout - Once the dependencies downloaded, run cli53 $ bin/cli53 +If the step bin/buildout fails with the cryptic error message +'pkg_resources.DistributionNotFound: distribute', then you might try +bootstrap.py with distribute instead of setuptools: + + $ python bootstrap.py --distribute + $ bin/buildout You need to set your Amazon credentials in the environment as AWS_ACCESS_KEY_ID -and AWS_SECRET_ACCESS_KEY. +and AWS_SECRET_ACCESS_KEY or configure them in ~/.boto. For more information see: +http://code.google.com/p/boto/wiki/BotoConfig Caveats ------- From 7dda7e6cfb8913192e8279cc65c8f5e8b52c88e1 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Wed, 10 Aug 2011 19:12:41 +0100 Subject: [PATCH 34/46] Use env for python path. Closes #13 --- src/cli53/cli53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index 4d34d3c8..a66bd13a 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # cli53 # Command line script to administer the Amazon Route 53 dns service From 194e0eeb2751bfd20d86259183fe52287f85e482 Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Wed, 10 Aug 2011 19:23:19 +0100 Subject: [PATCH 35/46] Fix for quoting TXT records. Closes #5 --- src/cli53/cli53.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index af00a840..6a03b573 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -200,6 +200,12 @@ def convert(self, info, rrsets, z=None): return z re_quoted = re.compile(r'^".*"$') +re_backslash = re.compile(r'\\(.)') +def unquote(v): + v = v[1:-1] + v = re_backslash.sub('\\1', v) + return v + def _create_rdataset(rtype, ttl, values): rdataset = dns.rdataset.Rdataset(dns.rdataclass.IN, dns.rdatatype.from_text(rtype)) rdataset.ttl = ttl @@ -238,7 +244,7 @@ def _create_rdataset(rtype, ttl, values): rdtype = SRV(dns.rdataclass.IN, dns.rdatatype.SRV, value) elif rtype == 'TXT': if re_quoted.match(value): - value = value[1:-1] + value = unquote(value) rdtype = dns.rdtypes.ANY.TXT.TXT(dns.rdataclass.ANY, dns.rdatatype.TXT, [value]) else: raise ValueError, 'record type %s not handled' % rtype From 3b468b70020b20e7aa9d3cd1e81dd851d2ae228b Mon Sep 17 00:00:00 2001 From: Barnaby Gray Date: Wed, 10 Aug 2011 19:35:01 +0100 Subject: [PATCH 36/46] Export CNAMES with full stop qualifier. Closes #18 --- src/cli53/cli53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index 6a03b573..166cdcc0 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -216,7 +216,7 @@ def _create_rdataset(rtype, ttl, values): elif rtype == 'AAAA': rdtype = dns.rdtypes.IN.AAAA.AAAA(dns.rdataclass.IN, dns.rdatatype.AAAA, value) elif rtype == 'CNAME': - rdtype = CNAME(dns.rdataclass.ANY, dns.rdatatype.CNAME, value) + rdtype = CNAME(dns.rdataclass.ANY, dns.rdatatype.CNAME, value.rstrip('.')+'.') elif rtype == 'SOA': mname, rname, serial, refresh, retry, expire, minimum = value.split() mname = dns.name.from_text(mname) From b49edf6fb78db1db73b1efd455f7af40119cc0ec Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Tue, 30 Aug 2011 16:00:47 -0700 Subject: [PATCH 37/46] add naive support for bind-esque directives --- src/cli53/cli53.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index 166cdcc0..4e5539aa 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -256,23 +256,35 @@ def cmd_xml(args): re_dos = re.compile('\r\n$') re_origin = re.compile(r'\$ORIGIN (\S+)') +re_include = re.compile(r'\$INCLUDE (\S+)') def cmd_import(args): text = [] - for line in args.file: - line = re_dos.sub('\n', line) - text.append(line) + + def file_parse(zonefile): + for line in zonefile: + line = re_dos.sub('\n', line) + inc = re_include.search(line) + if inc: + incfile = open(inc.group(1)) + file_parse(incfile) + incfile.close() + else: + text.append(line) + + file_parse(args.file) + text = ''.join(text) - + m = re_origin.search(text) if not m: raise Exception, 'Could not find origin' origin = m.group(1) - + zone = dns.zone.from_text(text, origin=origin, check_origin=True) old_zone = None if args.replace: old_zone = _get_records(args) - + f = BindToR53Formatter() for xml in f.create_all(zone, old_zone=old_zone, exclude=is_root_soa_or_ns): ret = r53.change_rrsets(args.zone, xml) @@ -280,7 +292,7 @@ def cmd_import(args): wait_for_sync(ret) else: pprint(ret.ChangeResourceRecordSetsResponse) - + re_zone_id = re.compile('^[A-Z0-9]+$') def Zone(zone): if re_zone_id.match(zone): From 81d5c3203b1045c003b62c1ae1ffe71faae41b74 Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Wed, 31 Aug 2011 12:25:33 -0700 Subject: [PATCH 38/46] add ability to dump generated xml to stdout --- src/cli53/cli53.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index 4e5539aa..d8bf594c 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -287,11 +287,14 @@ def file_parse(zonefile): f = BindToR53Formatter() for xml in f.create_all(zone, old_zone=old_zone, exclude=is_root_soa_or_ns): - ret = r53.change_rrsets(args.zone, xml) - if args.wait: - wait_for_sync(ret) + if args.dump: + print xml else: - pprint(ret.ChangeResourceRecordSetsResponse) + ret = r53.change_rrsets(args.zone, xml) + if args.wait: + wait_for_sync(ret) + else: + pprint(ret.ChangeResourceRecordSetsResponse) re_zone_id = re.compile('^[A-Z0-9]+$') def Zone(zone): @@ -464,6 +467,7 @@ def main(): parser_import.add_argument('-r', '--replace', action='store_true', help='replace all existing records (use with care!)') parser_import.add_argument('-f', '--file', type=argparse.FileType('r'), help='bind file') parser_import.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') + parser_import.add_argument('--dump', action='store_true', help='dump xml to stdout') parser_import.set_defaults(func=cmd_import) parser_create = subparsers.add_parser('create', help='create a hosted zone') From 5a37145316d3429109a49830f39a3883abaa91e0 Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Wed, 31 Aug 2011 22:41:55 -0700 Subject: [PATCH 39/46] expand dump options to print either the zone xml or the changeset xml --- src/cli53/cli53.py | 118 ++++++++++++++++++++++++++++----------------- 1 file changed, 73 insertions(+), 45 deletions(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index d8bf594c..d8e67e71 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -10,7 +10,7 @@ from time import sleep # needs latest boto from github: http://github.com/boto/boto -# git clone git://github.com/boto/boto +# git clone git://github.com/boto/boto try: import boto.route53, boto.jsonresponse, boto.exception except ImportError: @@ -97,15 +97,15 @@ def pprint(obj, findent='', indent=''): def cmd_list(args): ret = r53.get_all_hosted_zones() pprint(ret.ListHostedZonesResponse) - + def cmd_info(args): ret = r53.get_hosted_zone(args.zone) pprint(ret.GetHostedZoneResponse) - + def text_element(parent, name, text): el = et.SubElement(parent, name) el.text = text - + def is_root_soa_or_ns(name, rdataset): rt = dns.rdatatype.to_text(rdataset.rdtype) return (rt in ('SOA', 'NS') and name.to_text() == '@') @@ -116,7 +116,7 @@ def paginate(iterable, size): while item: yield item item = list(itertools.islice(it, size)) - + class BindToR53Formatter(object): def _build_list(self, zone, exclude=None): li = [] @@ -125,7 +125,7 @@ def _build_list(self, zone, exclude=None): if not exclude or not exclude(name, rdataset): li.append((name, rdataset)) return li - + def create_all(self, zone, old_zone=None, exclude=None): creates = self._build_list(zone, exclude) deletes = [] @@ -133,32 +133,52 @@ def create_all(self, zone, old_zone=None, exclude=None): deletes = self._build_list(old_zone, exclude) return self._xml_changes(zone, creates=creates, deletes=deletes) + def dump_xml(self, zone, exclude=None): + resource_record_sets = et.Element('ResourceRecordSets', + xmlns=boto.route53.Route53Connection.XMLNameSpace) + + for rrname in zone.keys(): + for rdataset in zone[rrname].rdatasets: + resource_record_set = et.SubElement(resource_record_sets, 'ResourceRecordSet') + text_element(resource_record_set, 'Name', rrname.derelativize(zone.origin).to_text()) + text_element(resource_record_set, 'Type', dns.rdatatype.to_text(rdataset.rdtype)) + text_element(resource_record_set, 'TTL', str(rdataset.ttl)) + resource_records = et.SubElement(resource_record_set, 'ResourceRecords') + for rdtype in rdataset.items: + resource_record = et.SubElement(resource_records, 'ResourceRecord') + text_element(resource_record, 'Value', rdtype.to_text(origin=zone.origin, relativize=False)) + + out = StringIO() + et.ElementTree(resource_record_sets).write(out) + return out.getvalue() + + def delete_all(self, zone, exclude=None): return self._xml_changes(zone, deletes=self._build_list(zone, exclude)) - + def create_record(self, zone, name, rdataset): return self._xml_changes(zone, creates=[(name,rdataset)]) - + def delete_record(self, zone, name, rdataset): return self._xml_changes(zone, deletes=[(name,rdataset)]) - + def replace_record(self, zone, name, rdataset_old, rdataset_new): return self._xml_changes(zone, creates=[(name,rdataset_new)], deletes=[(name,rdataset_old)]) - + def _xml_changes(self, zone, creates=[], deletes=[]): for page in paginate(self._iter_changes(creates, deletes), 100): yield self._batch_change(zone, page) - + def _iter_changes(self, creates, deletes): for chg, rdatasets in (('DELETE', deletes), ('CREATE', creates)): for name, rdataset in rdatasets: yield chg, name, rdataset - + def _batch_change(self, zone, chgs): root = et.Element('ChangeResourceRecordSetsRequest', xmlns=boto.route53.Route53Connection.XMLNameSpace) change_batch = et.SubElement(root, 'ChangeBatch') changes = et.SubElement(change_batch, 'Changes') - + for chg, name, rdataset in chgs: change = et.SubElement(changes, 'Change') text_element(change, 'Action', chg) @@ -170,21 +190,21 @@ def _batch_change(self, zone, chgs): for rdtype in rdataset.items: rr = et.SubElement(rrs, 'ResourceRecord') text_element(rr, 'Value', rdtype.to_text(origin=zone.origin, relativize=False)) - + out = StringIO() et.ElementTree(root).write(out) return out.getvalue() - + class R53ToBindFormatter(object): def get_all_rrsets(self, r53, ghz, zone): rrsets = r53.get_all_rrsets(zone, maxitems=10) return self.convert(ghz, rrsets) - + def convert(self, info, rrsets, z=None): if not z: origin = info.HostedZone.Name z = dns.zone.Zone(dns.name.from_text(origin)) - + for rrset in rrsets: name = rrset.name if '\\052' in name: @@ -196,9 +216,9 @@ def convert(self, info, rrsets, z=None): rdataset = _create_rdataset(rtype, ttl, rrset.resource_records) node = z.get_node(name, create=True) node.replace_rdataset(rdataset) - + return z - + re_quoted = re.compile(r'^".*"$') re_backslash = re.compile(r'\\(.)') def unquote(v): @@ -250,10 +270,10 @@ def _create_rdataset(rtype, ttl, values): raise ValueError, 'record type %s not handled' % rtype rdataset.items.append(rdtype) return rdataset - + def cmd_xml(args): print 'This functionality is no longer available due to changes in the boto library.' - + re_dos = re.compile('\r\n$') re_origin = re.compile(r'\$ORIGIN (\S+)') re_include = re.compile(r'\$INCLUDE (\S+)') @@ -281,13 +301,20 @@ def file_parse(zonefile): origin = m.group(1) zone = dns.zone.from_text(text, origin=origin, check_origin=True) + + if args.dump: + f = BindToR53Formatter() + xml = f.dump_xml(zone) + print xml + return + old_zone = None if args.replace: old_zone = _get_records(args) f = BindToR53Formatter() for xml in f.create_all(zone, old_zone=old_zone, exclude=is_root_soa_or_ns): - if args.dump: + if args.dumpchg: print xml else: ret = r53.change_rrsets(args.zone, xml) @@ -309,7 +336,7 @@ def Zone(zone): raise ArgumentTypeError, 'Zone %r is ambiguous (matches: %s), please specify ID' % (zone, ', '.join(hzs)) else: raise ArgumentTypeError, 'Zone %r not found' % zone - + def _get_records(args): info = r53.get_hosted_zone(args.zone) f = R53ToBindFormatter() @@ -319,14 +346,14 @@ def cmd_export(args): zone = _get_records(args) print '$ORIGIN %s' % zone.origin.to_text() zone.to_file(sys.stdout, relativize=not args.full) - + def cmd_create(args): ret = r53.create_hosted_zone(args.zone) if args.wait: wait_for_sync(ret) else: pprint(ret.CreateHostedZoneResponse) - + def cmd_delete(args): ret = r53.delete_hosted_zone(args.zone) if hasattr(ret, 'ErrorResponse'): @@ -345,7 +372,7 @@ def find_key_nonrecursive(adict, key): for k, v in d.iteritems(): if isinstance(v, dict): stack.append(v) - + def wait_for_sync(obj): waiting = 1 id = find_key_nonrecursive(obj, 'Id') @@ -363,7 +390,7 @@ def wait_for_sync(obj): sleep(1) print "completed" pprint(ret.GetChangeResponse) - + def cmd_rrcreate(args): zone = _get_records(args) name = dns.name.from_text(args.rr, zone.origin) @@ -405,11 +432,11 @@ def cmd_rrdelete(args): if args.type == dns.rdatatype.to_text(rds.rdtype) or not args.type: rdataset = rds break - + if not rdataset: print 'Record not found: %s, type: %s' % (args.rr, args.type) return - + print 'Deleting %s %s...' % (args.rr, dns.rdatatype.to_text(rds.rdtype)) f = BindToR53Formatter() @@ -422,7 +449,7 @@ def cmd_rrdelete(args): pprint(ret.ChangeResourceRecordSetsResponse) else: print 'Record not found: %s' % args.rr - + def cmd_rrpurge(args): zone = _get_records(args) f = BindToR53Formatter() @@ -432,7 +459,7 @@ def cmd_rrpurge(args): wait_for_sync(ret) else: pprint(ret.ChangeResourceRecordSetsResponse) - + def cmd_rrlist(args): zone = _get_records(args) print '\t'.join(["host","ttl","cls","type","data"]) @@ -443,43 +470,44 @@ def main(): connection = boto.route53.Route53Connection() parser = argparse.ArgumentParser(description='route53 command line tool') subparsers = parser.add_subparsers(help='sub-command help') - + supported_rtypes = ('A', 'AAAA', 'CNAME', 'SOA', 'NS', 'MX', 'PTR', 'SPF', 'SRV', 'TXT') - + parser_list = subparsers.add_parser('list', help='list hosted zones') parser_list.set_defaults(func=cmd_list) - + parser_list = subparsers.add_parser('info', help='get details of a hosted zone') parser_list.add_argument('zone', type=Zone, help='zone name') parser_list.set_defaults(func=cmd_info) - + parser_describe = subparsers.add_parser('xml') parser_describe.add_argument('zone', type=Zone, help='zone name') parser_describe.set_defaults(func=cmd_xml) - + parser_describe = subparsers.add_parser('export', help='export dns in bind format') parser_describe.add_argument('zone', type=Zone, help='zone name') parser_describe.add_argument('--full', action='store_true', help='export prefixes as full names') parser_describe.set_defaults(func=cmd_export) - + parser_import = subparsers.add_parser('import', help='import dns in bind format') parser_import.add_argument('zone', type=Zone, help='zone name') parser_import.add_argument('-r', '--replace', action='store_true', help='replace all existing records (use with care!)') parser_import.add_argument('-f', '--file', type=argparse.FileType('r'), help='bind file') parser_import.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') - parser_import.add_argument('--dump', action='store_true', help='dump xml to stdout') + parser_import.add_argument('--dump', action='store_true', help='dump zone file in xml format to stdout') + parser_import.add_argument('--dumpchg', action='store_true', help='dump xml change request to stdout') parser_import.set_defaults(func=cmd_import) - + parser_create = subparsers.add_parser('create', help='create a hosted zone') parser_create.add_argument('zone', help='zone name') parser_create.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_create.set_defaults(func=cmd_create) - + parser_delete = subparsers.add_parser('delete', help='delete a hosted zone') parser_delete.add_argument('zone', type=Zone, help='zone name') parser_delete.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_delete.set_defaults(func=cmd_delete) - + parser_rrcreate = subparsers.add_parser('rrcreate', help='create a resource record') parser_rrcreate.add_argument('zone', type=Zone, help='zone name') parser_rrcreate.add_argument('rr', help='resource record') @@ -489,26 +517,26 @@ def main(): parser_rrcreate.add_argument('-r', '--replace', action='store_true', help='replace any existing record') parser_rrcreate.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_rrcreate.set_defaults(func=cmd_rrcreate) - + parser_rrdelete = subparsers.add_parser('rrdelete', help='delete a resource record') parser_rrdelete.add_argument('zone', type=Zone, help='zone name') parser_rrdelete.add_argument('rr', help='resource record') parser_rrdelete.add_argument('type', nargs='?', choices=supported_rtypes, help='resource record type') parser_rrdelete.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_rrdelete.set_defaults(func=cmd_rrdelete) - + parser_rrpurge = subparsers.add_parser('rrpurge', help='purge all resource records') parser_rrpurge.add_argument('zone', type=Zone, help='zone name') parser_rrpurge.add_argument('--confirm', required=True, action='store_true', help='confirm you definitely want to do this!') parser_rrpurge.add_argument('--wait', action='store_true', default=False, help='wait for changes to become live before exiting (default: false)') parser_rrpurge.set_defaults(func=cmd_rrpurge) - + parser_rrlist = subparsers.add_parser('rrlist', help='list all resource records') parser_rrlist.add_argument('zone', type=Zone, help='zone name') parser_rrlist.set_defaults(func=cmd_rrlist) args = parser.parse_args() args.func(args) - + if __name__=='__main__': main() From 1d9b8ed824150b95514359aa104e37fe0d9ef37c Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Wed, 7 Sep 2011 13:37:02 -0700 Subject: [PATCH 40/46] fix typo in parser_info setup --- src/cli53/cli53.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index d8e67e71..41edac23 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -476,9 +476,9 @@ def main(): parser_list = subparsers.add_parser('list', help='list hosted zones') parser_list.set_defaults(func=cmd_list) - parser_list = subparsers.add_parser('info', help='get details of a hosted zone') - parser_list.add_argument('zone', type=Zone, help='zone name') - parser_list.set_defaults(func=cmd_info) + parser_info = subparsers.add_parser('info', help='get details of a hosted zone') + parser_info.add_argument('zone', type=Zone, help='zone name') + parser_info.set_defaults(func=cmd_info) parser_describe = subparsers.add_parser('xml') parser_describe.add_argument('zone', type=Zone, help='zone name') From cd2571860799cd8f58e6d3d02a1bccdad9d7c019 Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Sat, 8 Oct 2011 22:53:06 -0700 Subject: [PATCH 41/46] Support AWS Alias records. Add an ugly shim in the xml dump output to reformat certain TXT records as amazon R53 alias records. --- src/cli53/cli53.py | 71 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 9 deletions(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index 41edac23..ece80a23 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -134,19 +134,72 @@ def create_all(self, zone, old_zone=None, exclude=None): return self._xml_changes(zone, creates=creates, deletes=deletes) def dump_xml(self, zone, exclude=None): - resource_record_sets = et.Element('ResourceRecordSets', - xmlns=boto.route53.Route53Connection.XMLNameSpace) + re_awsalias = re.compile(r'^AWSALIAS') + # preprocess; this is annoying but necessary to support our little + # TXT record shim: doing it inside dnspython is just painful + rr_data = {} for rrname in zone.keys(): + rr_name = rrname.derelativize(zone.origin).to_text() + rr_data[rr_name] = {} for rdataset in zone[rrname].rdatasets: - resource_record_set = et.SubElement(resource_record_sets, 'ResourceRecordSet') - text_element(resource_record_set, 'Name', rrname.derelativize(zone.origin).to_text()) - text_element(resource_record_set, 'Type', dns.rdatatype.to_text(rdataset.rdtype)) - text_element(resource_record_set, 'TTL', str(rdataset.ttl)) - resource_records = et.SubElement(resource_record_set, 'ResourceRecords') + rr_type = dns.rdatatype.to_text(rdataset.rdtype) + rr_data[rr_name][rr_type] = {} + rr_data[rr_name][rr_type]['TTL'] = str(rdataset.ttl) + rr_data[rr_name][rr_type]['RRS'] = set() for rdtype in rdataset.items: - resource_record = et.SubElement(resource_records, 'ResourceRecord') - text_element(resource_record, 'Value', rdtype.to_text(origin=zone.origin, relativize=False)) + rr_data[rr_name][rr_type]['RRS'].add(rdtype.to_text(origin=zone.origin, + relativize=False)) + + # now deal with the ugliness of aws alias records + for rr_name in rr_data: + # first, convert any AWSALIAS txt records into A records + if 'TXT' in rr_data[rr_name]: + rr_vals_to_delete = set() + for rr_value in rr_data[rr_name]['TXT']['RRS']: + if re_awsalias.search(unquote(rr_value)): + (_, hosted_zone_id, dns_name) = unquote(rr_value).split(':') + # remove the awsalias from the TXT record set + rr_vals_to_delete.add(rr_value) + # add as an A record with an alias target + if 'A' not in rr_data[rr_name]: + rr_data[rr_name]['A'] = {} + rr_data[rr_name]['A']['AliasTarget'] = {} + rr_data[rr_name]['A']['AliasTarget']['HostedZoneId'] = hosted_zone_id + rr_data[rr_name]['A']['AliasTarget']['DNSName'] = dns_name + for rr_value in rr_vals_to_delete: + rr_data[rr_name]['TXT']['RRS'].remove(rr_value) + # if we've emptied the TXT set, delete it + if not rr_data[rr_name]['TXT']['RRS']: + del rr_data[rr_name]['TXT'] + # now make sure there's no existing A record for that RR + if 'A' in rr_data[rr_name]: + if 'RRS' in rr_data[rr_name]['A'] and 'AliasTarget' in rr_data[rr_name]['A']: + raise ValueError( + 'You cannot have both a static A record and an AWSALIAS' + ' at the same RR node: %s' % rr_name) + + # now spit it all back out as XML + resource_record_sets = et.Element('ResourceRecordSets', + xmlns=boto.route53.Route53Connection.XMLNameSpace) + + for rr_name in rr_data: + for rr_type in rr_data[rr_name]: + resource_record_set = et.SubElement(resource_record_sets, 'ResourceRecordSet') + text_element(resource_record_set, 'Name', rr_name) + text_element(resource_record_set, 'Type', rr_type) + if 'AliasTarget' in rr_data[rr_name][rr_type]: + alias_target = et.SubElement(resource_record_set, 'AliasTarget') + text_element(alias_target, 'HostedZoneId', + rr_data[rr_name][rr_type]['AliasTarget']['HostedZoneId']) + text_element(alias_target, 'DNSName', + rr_data[rr_name][rr_type]['AliasTarget']['DNSName']) + else: + text_element(resource_record_set, 'TTL', rr_data[rr_name][rr_type]['TTL']) + resource_records = et.SubElement(resource_record_set, 'ResourceRecords') + for rr_value in rr_data[rr_name][rr_type]['RRS']: + resource_record = et.SubElement(resource_records, 'ResourceRecord') + text_element(resource_record, 'Value', rr_value) out = StringIO() et.ElementTree(resource_record_sets).write(out) From d3f02b3670c40696bcb1705d81368ecadb205cd9 Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Sat, 8 Oct 2011 23:11:34 -0700 Subject: [PATCH 42/46] use lists instead of sets when assembling xml output r53 considers resourcerecord order to be important, sigh. --- src/cli53/cli53.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index ece80a23..d30e8c33 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -146,21 +146,21 @@ def dump_xml(self, zone, exclude=None): rr_type = dns.rdatatype.to_text(rdataset.rdtype) rr_data[rr_name][rr_type] = {} rr_data[rr_name][rr_type]['TTL'] = str(rdataset.ttl) - rr_data[rr_name][rr_type]['RRS'] = set() + rr_data[rr_name][rr_type]['RRS'] = [] for rdtype in rdataset.items: - rr_data[rr_name][rr_type]['RRS'].add(rdtype.to_text(origin=zone.origin, + rr_data[rr_name][rr_type]['RRS'].append(rdtype.to_text(origin=zone.origin, relativize=False)) # now deal with the ugliness of aws alias records for rr_name in rr_data: # first, convert any AWSALIAS txt records into A records if 'TXT' in rr_data[rr_name]: - rr_vals_to_delete = set() + rr_vals_to_delete = [] for rr_value in rr_data[rr_name]['TXT']['RRS']: if re_awsalias.search(unquote(rr_value)): (_, hosted_zone_id, dns_name) = unquote(rr_value).split(':') # remove the awsalias from the TXT record set - rr_vals_to_delete.add(rr_value) + rr_vals_to_delete.append(rr_value) # add as an A record with an alias target if 'A' not in rr_data[rr_name]: rr_data[rr_name]['A'] = {} @@ -168,7 +168,8 @@ def dump_xml(self, zone, exclude=None): rr_data[rr_name]['A']['AliasTarget']['HostedZoneId'] = hosted_zone_id rr_data[rr_name]['A']['AliasTarget']['DNSName'] = dns_name for rr_value in rr_vals_to_delete: - rr_data[rr_name]['TXT']['RRS'].remove(rr_value) + del(rr_data[rr_name]['TXT']['RRS'][ + rr_data[rr_name]['TXT']['RRS'].index(rr_value)]) # if we've emptied the TXT set, delete it if not rr_data[rr_name]['TXT']['RRS']: del rr_data[rr_name]['TXT'] From 323ac165ee89078bf07f6dd763ed2e9739334d1b Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Tue, 5 May 2015 13:44:46 -0700 Subject: [PATCH 43/46] evaluate_health is now required --- src/cli53/cli53.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index d30e8c33..163e6b04 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -158,7 +158,7 @@ def dump_xml(self, zone, exclude=None): rr_vals_to_delete = [] for rr_value in rr_data[rr_name]['TXT']['RRS']: if re_awsalias.search(unquote(rr_value)): - (_, hosted_zone_id, dns_name) = unquote(rr_value).split(':') + (_, hosted_zone_id, dns_name, eval_health) = unquote(rr_value).split(':') # remove the awsalias from the TXT record set rr_vals_to_delete.append(rr_value) # add as an A record with an alias target @@ -167,6 +167,7 @@ def dump_xml(self, zone, exclude=None): rr_data[rr_name]['A']['AliasTarget'] = {} rr_data[rr_name]['A']['AliasTarget']['HostedZoneId'] = hosted_zone_id rr_data[rr_name]['A']['AliasTarget']['DNSName'] = dns_name + rr_data[rr_name]['A']['AliasTarget']['EvaluateTargetHealth'] = eval_health for rr_value in rr_vals_to_delete: del(rr_data[rr_name]['TXT']['RRS'][ rr_data[rr_name]['TXT']['RRS'].index(rr_value)]) From 9ecf40d3d8148a5a16ae713eef6aafe0134049c9 Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Tue, 5 May 2015 13:55:26 -0700 Subject: [PATCH 44/46] spit evaluatetargethealth back out in xml --- src/cli53/cli53.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index 163e6b04..9bfafce0 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -196,6 +196,8 @@ def dump_xml(self, zone, exclude=None): rr_data[rr_name][rr_type]['AliasTarget']['HostedZoneId']) text_element(alias_target, 'DNSName', rr_data[rr_name][rr_type]['AliasTarget']['DNSName']) + text_element(alias_target, 'DNSName', + rr_data[rr_name][rr_type]['AliasTarget']['EvaluateTargetHealth']) else: text_element(resource_record_set, 'TTL', rr_data[rr_name][rr_type]['TTL']) resource_records = et.SubElement(resource_record_set, 'ResourceRecords') From 1c67b2ba8459d50f1682d95954ee25f26914ae0e Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Mon, 11 May 2015 08:46:49 -0700 Subject: [PATCH 45/46] elementtree is long since dead --- buildout.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildout.cfg b/buildout.cfg index 50f7b1ab..2bc7fe53 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -11,7 +11,7 @@ eggs = cli53 [versions] boto = 2.0b4 dnspython = 1.9.4 -elementtree = 1.2.6-20050316 +lxml = 3.4.4 uuid = 1.30 diff --git a/setup.py b/setup.py index 8cb2c19d..0dbda38c 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ install_requires=[ 'boto', 'dnspython', - 'elementtree', + 'lxml', 'uuid', ], entry_points={ From 2b4c978c5f6e346e3b7ea448bf360bdce8f528df Mon Sep 17 00:00:00 2001 From: "Nathan J. Mehl" Date: Mon, 11 May 2015 08:56:25 -0700 Subject: [PATCH 46/46] fix EvaluateTargetHealth output --- src/cli53/cli53.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli53/cli53.py b/src/cli53/cli53.py index 9bfafce0..681e5007 100755 --- a/src/cli53/cli53.py +++ b/src/cli53/cli53.py @@ -196,7 +196,7 @@ def dump_xml(self, zone, exclude=None): rr_data[rr_name][rr_type]['AliasTarget']['HostedZoneId']) text_element(alias_target, 'DNSName', rr_data[rr_name][rr_type]['AliasTarget']['DNSName']) - text_element(alias_target, 'DNSName', + text_element(alias_target, 'EvaluateTargetHealth', rr_data[rr_name][rr_type]['AliasTarget']['EvaluateTargetHealth']) else: text_element(resource_record_set, 'TTL', rr_data[rr_name][rr_type]['TTL'])