From d76684c4fee2ccaf5baba4ced84c3b8fbcd508b1 Mon Sep 17 00:00:00 2001 From: Tony Duckles Date: Sun, 22 Jan 2012 16:30:12 -0600 Subject: [PATCH] Add svn2svn specific changes --- .gitignore | 3 + svn2svn/shell.py | 216 +++++++++++++++++++++++++++++++++++++++++++ svn2svn/svnclient.py | 92 ++++++++++-------- svn2svn/ui.py | 51 +++++----- 4 files changed, 296 insertions(+), 66 deletions(-) create mode 100644 svn2svn/shell.py diff --git a/.gitignore b/.gitignore index 132393a..387da60 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +*.pyc +*.pyo tests/_repo* tests/_wc* tests/_dup_wc +*.swp diff --git a/svn2svn/shell.py b/svn2svn/shell.py new file mode 100644 index 0000000..9069b21 --- /dev/null +++ b/svn2svn/shell.py @@ -0,0 +1,216 @@ +""" Shell functions """ + +from svn2svn import ui +from svn2svn.errors import ExternalCommandFailed + +import os +import locale +from datetime import datetime +import time +from subprocess import Popen, PIPE, STDOUT +import shutil +import stat +import sys +import traceback +import re + +try: + import commands +except ImportError: + commands = None + + +# Windows compatibility code by Bill Baxter +if os.name == "nt": + def find_program(name): + """ + Find the name of the program for Popen. + Windows is finnicky about having the complete file name. Popen + won't search the %PATH% for you automatically. + (Adapted from ctypes.find_library) + """ + # See MSDN for the REAL search order. + base, ext = os.path.splitext(name) + if ext: + exts = [ext] + else: + exts = ['.bat', '.exe'] + for directory in os.environ['PATH'].split(os.pathsep): + for e in exts: + fname = os.path.join(directory, base + e) + if os.path.exists(fname): + return fname + return name +else: + def find_program(name): + """ + Find the name of the program for Popen. + On Unix, popen isn't picky about having absolute paths. + """ + return name + + +def _rmtree_error_handler(func, path, exc_info): + """ + Error handler for rmtree. Helps removing the read-only protection under + Windows (and others?). + Adapted from http://www.proaxis.com/~darkwing/hot-backup.py + and http://patchwork.ozlabs.org/bazaar-ng/patch?id=4243 + """ + if func in (os.remove, os.rmdir) and os.path.exists(path): + # Change from read-only to writeable + os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE) + func(path) + else: + # something else must be wrong... + raise + +def rmtree(path): + """ + Wrapper around shutil.rmtree(), to provide more error-resistent behaviour. + """ + return shutil.rmtree(path, False, _rmtree_error_handler) + + +locale_encoding = locale.getpreferredencoding() + +def get_encoding(): + return locale_encoding + +def shell_quote(s): + global _debug_showcmd + if _debug_showcmd: + # If showing OS commands being executed, don't wrap "safe" strings in quotes. + if re.compile('^[A-Za-z0-9=-]+$').match(s): + return s + if os.name == "nt": + q = '"' + else: + q = "'" + return q + s.replace('\\', '\\\\').replace("'", "'\"'\"'") + q + +def _run_raw_command(cmd, args, fail_if_stderr=False): + cmd_string = "%s %s" % (cmd, " ".join(map(shell_quote, args))) + ui.status("* %s", cmd_string, level=ui.DEBUG) + try: + pipe = Popen([cmd] + args, executable=cmd, stdout=PIPE, stderr=PIPE) + except OSError: + etype, value = sys.exc_info()[:2] + raise ExternalCommandFailed( + "Failed running external program: %s\nError: %s" + % (cmd_string, "".join(traceback.format_exception_only(etype, value)))) + out, err = pipe.communicate() + if "nothing changed" == out.strip(): # skip this error + return out + if pipe.returncode != 0 or (fail_if_stderr and err.strip()): + raise ExternalCommandFailed( + "External program failed (return code %d): %s\n%s\n%s" + % (pipe.returncode, cmd_string, err, out)) + return out + +def _run_raw_shell_command(cmd): + ui.status("* %s", cmd, level=ui.DEBUG) + st, out = commands.getstatusoutput(cmd) + if st != 0: + raise ExternalCommandFailed( + "External program failed with non-zero return code (%d): %s\n%s" + % (st, cmd, out)) + return out + +def run_command(cmd, args=None, bulk_args=None, encoding=None, fail_if_stderr=False): + """ + Run a command without using the shell. + """ + args = args or [] + bulk_args = bulk_args or [] + def _transform_arg(a): + if isinstance(a, unicode): + a = a.encode(encoding or locale_encoding or 'UTF-8') + elif not isinstance(a, str): + a = str(a) + return a + + cmd = find_program(cmd) + if not bulk_args: + return _run_raw_command(cmd, map(_transform_arg, args), fail_if_stderr) + # If one of bulk_args starts with a dash (e.g. '-foo.php'), + # svn will take this as an option. Adding '--' ends the search for + # further options. + for a in bulk_args: + if a.strip().startswith('-'): + args.append("--") + break + max_args_num = 254 + i = 0 + out = "" + while i < len(bulk_args): + stop = i + max_args_num - len(args) + sub_args = [] + for a in bulk_args[i:stop]: + sub_args.append(_transform_arg(a)) + out += _run_raw_command(cmd, args + sub_args, fail_if_stderr) + i = stop + return out + +def run_shell_command(cmd, args=None, bulk_args=None, encoding=None): + """ + Run a shell command, properly quoting and encoding arguments. + Probably only works on Un*x-like systems. + """ + def _quote_arg(a): + if isinstance(a, unicode): + a = a.encode(encoding or locale_encoding) + elif not isinstance(a, str): + a = str(a) + return shell_quote(a) + + if args: + cmd += " " + " ".join(_quote_arg(a) for a in args) + max_args_num = 254 + i = 0 + out = "" + if not bulk_args: + return _run_raw_shell_command(cmd) + while i < len(bulk_args): + stop = i + max_args_num - len(args) + sub_args = [] + for a in bulk_args[i:stop]: + sub_args.append(_quote_arg(a)) + sub_cmd = cmd + " " + " ".join(sub_args) + out += _run_raw_shell_command(sub_cmd) + i = stop + return out + +def run_svn(args=None, bulk_args=None, fail_if_stderr=False, + mask_atsign=False): + """ + Run an SVN command, returns the (bytes) output. + """ + if mask_atsign: + # The @ sign in Subversion revers to a pegged revision number. + # SVN treats files with @ in the filename a bit special. + # See: http://stackoverflow.com/questions/1985203 + for idx in range(len(args)): + if "@" in args[idx] and args[idx][0] not in ("-", '"'): + args[idx] = "%s@" % args[idx] + if bulk_args: + for idx in range(len(bulk_args)): + if ("@" in bulk_args[idx] + and bulk_args[idx][0] not in ("-", '"')): + bulk_args[idx] = "%s@" % bulk_args[idx] + return run_command("svn", + args=args, bulk_args=bulk_args, fail_if_stderr=fail_if_stderr) + +def skip_dirs(paths, basedir="."): + """ + Skip all directories from path list, including symbolic links to real dirs. + """ + # NOTE: both tests are necessary (Cameron Hutchison's patch for symbolic + # links to directories) + return [p for p in paths + if not os.path.isdir(os.path.join(basedir, p)) + or os.path.islink(os.path.join(basedir, p))] + +def get_script_name(): + """Helper to return the name of the command line script that was called.""" + return os.path.basename(sys.argv[0]) diff --git a/svn2svn/svnclient.py b/svn2svn/svnclient.py index 031f119..449512c 100644 --- a/svn2svn/svnclient.py +++ b/svn2svn/svnclient.py @@ -1,7 +1,8 @@ +""" SVN client functions """ -from hgsvn import ui -from hgsvn.common import (run_svn, once_or_more) -from hgsvn.errors import EmptySVNLog +from svn2svn import ui +from svn2svn.shell import run_svn +from svn2svn.errors import EmptySVNLog import os import time @@ -19,11 +20,10 @@ except ImportError: except ImportError: from elementtree import ElementTree as ET - -svn_log_args = ['log', '--xml', '-v'] +svn_log_args = ['log', '--xml'] svn_info_args = ['info', '--xml'] svn_checkout_args = ['checkout', '-q'] -svn_status_args = ['status', '--xml', '--ignore-externals'] +svn_status_args = ['status', '--xml', '-v', '--ignore-externals'] _identity_table = "".join(map(chr, range(256))) _forbidden_xml_chars = "".join( @@ -61,8 +61,10 @@ def parse_svn_info_xml(xml_string): tree = ET.fromstring(xml_string) entry = tree.find('.//entry') d['url'] = entry.find('url').text + d['kind'] = entry.get('kind') d['revision'] = int(entry.get('revision')) d['repos_url'] = tree.find('.//repository/root').text + d['repos_uuid'] = tree.find('.//repository/uuid').text d['last_changed_rev'] = int(tree.find('.//commit').get('revision')) author_element = tree.find('.//commit/author') if author_element is not None: @@ -90,24 +92,31 @@ def parse_svn_log_xml(xml_string): author = entry.find('author') date = entry.find('date') msg = entry.find('msg') - # Issue 64 - modified to prevent crashes on svn log entries with "No author" d['author'] = author is not None and author.text or "No author" if date is not None: d['date'] = svn_date_to_timestamp(date.text) else: d['date'] = None - d['message'] = msg is not None and msg.text or "" - paths = d['changed_paths'] = [] - for path in entry.findall('.//path'): + d['message'] = msg is not None and msg.text.replace('\r\n', '\n').replace('\n\r', '\n').replace('\r', '\n') or "" + paths = [] + for path in entry.findall('.//paths/path'): copyfrom_rev = path.get('copyfrom-rev') if copyfrom_rev: copyfrom_rev = int(copyfrom_rev) paths.append({ 'path': path.text, + 'kind': path.get('kind'), 'action': path.get('action'), 'copyfrom_path': path.get('copyfrom-path'), 'copyfrom_revision': copyfrom_rev, }) + # Sort paths (i.e. into hierarchical order), so that process_svn_log_entry() + # can process actions in depth-first order. + d['changed_paths'] = sorted(paths, key=itemgetter('path')) + revprops = [] + for prop in entry.findall('.//revprops/property'): + revprops.append({ 'name': prop.get('name'), 'value': prop.text }) + d['revprops'] = revprops l.append(d) return l @@ -132,12 +141,18 @@ def parse_svn_status_xml(xml_string, base_dir=None, ignore_externals=False): if wc_status.get('item') == 'external': if ignore_externals: continue + status = wc_status.get('item') + revision = wc_status.get('revision') + if status == 'external': d['type'] = 'external' - elif wc_status.get('revision') is not None: + elif revision is not None: d['type'] = 'normal' else: d['type'] = 'unversioned' - d['status'] = wc_status.get('item') + d['status'] = status + d['revision'] = revision + d['props'] = wc_status.get('props') + d['copied'] = wc_status.get('copied') l.append(d) return l @@ -148,11 +163,10 @@ def get_svn_info(svn_url_or_wc, rev_number=None): Returns a dict as created by parse_svn_info_xml(). """ if rev_number is not None: - args = ['-r', rev_number] + args = ["-r", rev_number, svn_url_or_wc+"@"+str(rev_number)] else: - args = [] - xml_string = run_svn(svn_info_args + args + [svn_url_or_wc], - fail_if_stderr=True) + args = [svn_url_or_wc] + xml_string = run_svn(svn_info_args + args, fail_if_stderr=True) return parse_svn_info_xml(xml_string) def svn_checkout(svn_url, checkout_dir, rev_number=None): @@ -165,19 +179,27 @@ def svn_checkout(svn_url, checkout_dir, rev_number=None): args += [svn_url, checkout_dir] return run_svn(svn_checkout_args + args) -def run_svn_log(svn_url, rev_start, rev_end, limit, stop_on_copy=False): +def run_svn_log(svn_url_or_wc, rev_start, rev_end, limit, stop_on_copy=False, get_changed_paths=True, get_revprops=False): """ Fetch up to 'limit' SVN log entries between the given revisions. """ + args = [] if stop_on_copy: - args = ['--stop-on-copy'] - else: - args = [] - args += ['-r', '%s:%s' % (rev_start, rev_end), '--limit', limit, svn_url] + args += ['--stop-on-copy'] + if get_changed_paths: + args += ['-v'] + if get_revprops: + args += ['--with-all-revprops'] + url = str(svn_url_or_wc) + if rev_start != 'HEAD' and rev_end != 'HEAD': + args += ['-r', '%s:%s' % (rev_start, rev_end)] + if not "@" in svn_url_or_wc: + url = "%s@%s" % (svn_url_or_wc, str(max(rev_start, rev_end))) + args += ['--limit', str(limit), url] xml_string = run_svn(svn_log_args + args) return parse_svn_log_xml(xml_string) -def get_svn_status(svn_wc, quiet=False): +def get_svn_status(svn_wc, quiet=False, no_recursive=False): """ Get SVN status information about the given working copy. """ @@ -188,6 +210,8 @@ def get_svn_status(svn_wc, quiet=False): args += ['-q'] else: args += ['-v'] + if no_recursive: + args += ['-N'] xml_string = run_svn(svn_status_args + args) return parse_svn_status_xml(xml_string, svn_wc, ignore_externals=True) @@ -201,19 +225,17 @@ def get_svn_versioned_files(svn_wc): contents.append(e['path']) return contents - -def get_one_svn_log_entry(svn_url, rev_start, rev_end, stop_on_copy=False): +def get_one_svn_log_entry(svn_url, rev_start, rev_end, stop_on_copy=False, get_changed_paths=True, get_revprops=False): """ Get the first SVN log entry in the requested revision range. """ - entries = run_svn_log(svn_url, rev_start, rev_end, 1, stop_on_copy) + entries = run_svn_log(svn_url, rev_start, rev_end, 1, stop_on_copy, get_changed_paths, get_revprops) if entries: return entries[0] raise EmptySVNLog("No SVN log for %s between revisions %s and %s" % (svn_url, rev_start, rev_end)) - -def get_first_svn_log_entry(svn_url, rev_start, rev_end): +def get_first_svn_log_entry(svn_url, rev_start, rev_end, get_changed_paths=True): """ Get the first log entry after (or at) the given revision number in an SVN branch. By default the revision number is set to 0, which will give you the log @@ -223,21 +245,21 @@ def get_first_svn_log_entry(svn_url, rev_start, rev_end): a copy from another branch, inspect elements of the 'changed_paths' entry in the returned dictionary. """ - return get_one_svn_log_entry(svn_url, rev_start, rev_end, stop_on_copy=True) + return get_one_svn_log_entry(svn_url, rev_start, rev_end, stop_on_copy=True, get_changed_paths=True) -def get_last_svn_log_entry(svn_url, rev_start, rev_end): +def get_last_svn_log_entry(svn_url, rev_start, rev_end, get_changed_paths=True): """ - Get the last log entry before (or at) the given revision number in an SVN branch. + Get the last log entry before/at the given revision number in an SVN branch. By default the revision number is set to HEAD, which will give you the log entry corresponding to the latest commit in branch. """ - return get_one_svn_log_entry(svn_url, rev_end, rev_start, stop_on_copy=True) + return get_one_svn_log_entry(svn_url, rev_end, rev_start, stop_on_copy=True, get_changed_paths=True) log_duration_threshold = 10.0 log_min_chunk_length = 10 -def iter_svn_log_entries(svn_url, first_rev, last_rev, retry): +def iter_svn_log_entries(svn_url, first_rev, last_rev, stop_on_copy=False, get_changed_paths=True, get_revprops=False): """ Iterate over SVN log entries between first_rev and last_rev. @@ -250,10 +272,8 @@ def iter_svn_log_entries(svn_url, first_rev, last_rev, retry): while last_rev == "HEAD" or cur_rev <= last_rev: start_t = time.time() stop_rev = min(last_rev, cur_rev + chunk_length) - ui.status("Fetching %s SVN log entries starting from revision %d...", - chunk_length, cur_rev, level=ui.VERBOSE) - entries = once_or_more("Fetching SVN log", retry, run_svn_log, svn_url, - cur_rev, "HEAD", chunk_length) + entries = run_svn_log(svn_url, cur_rev, "HEAD", chunk_length, + stop_on_copy, get_changed_paths, get_revprops) duration = time.time() - start_t if not first_run: # skip first revision on subsequent runs, as it is overlapped diff --git a/svn2svn/ui.py b/svn2svn/ui.py index 7bc4ad1..9f4aa19 100644 --- a/svn2svn/ui.py +++ b/svn2svn/ui.py @@ -1,39 +1,28 @@ -# -*- coding: utf-8 -*- - -"""User interface functions.""" +""" User interface functions """ import os import sys -try: - # First try to import the Mercurial implementation. - import mercurial.ui - if getattr(mercurial.ui.ui(), 'termwidth', False): - termwidth = mercurial.ui.ui().termwidth - else: - from mercurial.util import termwidth -except ImportError: - # Fallback to local copy of Mercurial's implementation. - def termwidth(): - if 'COLUMNS' in os.environ: +def termwidth(): + if 'COLUMNS' in os.environ: + try: + return int(os.environ['COLUMNS']) + except ValueError: + pass + try: + import termios, array, fcntl + for dev in (sys.stdout, sys.stdin): try: - return int(os.environ['COLUMNS']) + fd = dev.fileno() + if not os.isatty(fd): + continue + arri = fcntl.ioctl(fd, termios.TIOCGWINSZ, '\0' * 8) + return array.array('h', arri)[1] except ValueError: pass - try: - import termios, array, fcntl - for dev in (sys.stdout, sys.stdin): - try: - fd = dev.fileno() - if not os.isatty(fd): - continue - arri = fcntl.ioctl(fd, termios.TIOCGWINSZ, '\0' * 8) - return array.array('h', arri)[1] - except ValueError: - pass - except ImportError: - pass - return 80 + except ImportError: + pass + return 80 # Log levels @@ -45,6 +34,7 @@ DEBUG = 30 # Configuration _level = DEFAULT +_debug_showcmd = False def status(msg, *args, **kwargs): @@ -87,5 +77,6 @@ def status(msg, *args, **kwargs): def update_config(options): """Update UI configuration.""" - global _level + global _level,_debug_showcmd _level = options.verbosity + _debug_showcmd = options.showcmd -- 2.45.2