svn2svn/svnclient.py

   1 """ SVN client functions """
   2
   3 from shell import run_svn
   4 from errors import EmptySVNLog
   5
   6 import os
   7 import time
   8 import calendar
   9 import operator
  10
  11 try:
  12     from xml.etree import cElementTree as ET
  13 except ImportError:
  14     try:
  15         from xml.etree import ElementTree as ET
  16     except ImportError:
  17         try:
  18             import cElementTree as ET
  19         except ImportError:
  20             from elementtree import ElementTree as ET
  21
  22 _identity_table = "".join(map(chr, range(256)))
  23 _forbidden_xml_chars = "".join(
  24     set(map(chr, range(32))) - set('\x09\x0A\x0D')
  25 )
  26
  27
  28 def strip_forbidden_xml_chars(xml_string):
  29     """
  30     Given an XML string, strips forbidden characters as per the XML spec.
  31     (these are all control characters except 0x9, 0xA and 0xD).
  32     """
  33     return xml_string.translate(_identity_table, _forbidden_xml_chars)
  34
  35
  36 def svn_date_to_timestamp(svn_date):
  37     """
  38     Parse an SVN date as read from the XML output and return the corresponding
  39     timestamp.
  40     """
  41     # Strip microseconds and timezone (always UTC, hopefully)
  42     # XXX there are various ISO datetime parsing routines out there,
  43     # cf. http://seehuhn.de/comp/pdate
  44     date = svn_date.split('.', 2)[0]
  45     time_tuple = time.strptime(date, "%Y-%m-%dT%H:%M:%S")
  46     return calendar.timegm(time_tuple)
  47
  48 def parse_svn_info_xml(xml_string):
  49     """
  50     Parse the XML output from an "svn info" command and extract useful information
  51     as a dict.
  52     """
  53     d = {}
  54     xml_string = strip_forbidden_xml_chars(xml_string)
  55     tree = ET.fromstring(xml_string)
  56     entry = tree.find('.//entry')
  57     d['url'] = entry.find('url').text
  58     d['kind'] = entry.get('kind')
  59     d['revision'] = int(entry.get('revision'))
  60     d['repos_url'] = tree.find('.//repository/root').text
  61     d['repos_uuid'] = tree.find('.//repository/uuid').text
  62     d['last_changed_rev'] = int(tree.find('.//commit').get('revision'))
  63     author_element = tree.find('.//commit/author')
  64     if author_element is not None:
  65         d['last_changed_author'] = author_element.text
  66     d['last_changed_date'] = svn_date_to_timestamp(tree.find('.//commit/date').text)
  67     return d
  68
  69 def get_kind(svn_repos_url, svn_path, svn_rev, action, paths):
  70     """
  71     Calculate the "kind"-type of a given URL in the SVN repo.
  72     """
  73     # By default, just do a simple "svn info" based on passed-in params.
  74     info_path = svn_path
  75     info_rev =  svn_rev
  76     if action == 'D':
  77         # For deletions, we can't do an "svn info" at this revision.
  78         # Need to trace ancestry backwards.
  79         parents = []
  80         for p in paths:
  81             # Build a list of any copy-from's in this log_entry that we're a child of.
  82             if p['kind'] == 'dir' and p['copyfrom_revision'] and svn_path.startswith(p['path']+"/"):
  83                 parents.append(p['path'])
  84         if parents:
  85             # Use the nearest copy-from'd parent
  86             parents.sort()
  87             parent = parents[len(parents)-1]
  88             for p in paths:
  89                 if parent == p['path']:
  90                     info_path = info_path.replace(p['path'], p['copyfrom_path'])
  91                     info_rev =  p['copyfrom_revision']
  92         else:
  93             # If no parent copy-from's, then we should be able to check this path in
  94             # the preceeding revision.
  95             info_rev -= 1
  96     info = get_svn_info(svn_repos_url+info_path, info_rev)
  97     return info['kind']
  98
  99 def parse_svn_log_xml(xml_string):
 100     """
 101     Parse the XML output from an "svn log" command and extract useful information
 102     as a list of dicts (one per log changeset).
 103     """
 104     l = []
 105     xml_string = strip_forbidden_xml_chars(xml_string)
 106     tree = ET.fromstring(xml_string)
 107     for entry in tree.findall('logentry'):
 108         d = {}
 109         d['revision'] = int(entry.get('revision'))
 110         # Some revisions don't have authors, most notably the first revision
 111         # in a repository.
 112         # logentry nodes targeting directories protected by path-based
 113         # authentication have no child nodes at all. We return an entry
 114         # in that case. Anyway, as it has no path entries, no further
 115         # processing will be made.
 116         author = entry.find('author')
 117         date = entry.find('date')
 118         msg = entry.find('msg')
 119         d['author'] = author is not None and author.text or "No author"
 120         d['date_raw'] = date.text if date is not None else None
 121         d['date'] = svn_date_to_timestamp(date.text) if date is not None else None
 122         d['message'] = msg is not None and msg.text and msg.text.replace('\r\n', '\n').replace('\n\r', '\n').replace('\r', '\n') or ""
 123         paths = []
 124         for path in entry.findall('.//paths/path'):
 125             copyfrom_rev = path.get('copyfrom-rev')
 126             if copyfrom_rev:
 127                 copyfrom_rev = int(copyfrom_rev)
 128             paths.append({
 129                 'path': path.text,
 130                 'kind': path.get('kind'),
 131                 'action': path.get('action'),
 132                 'copyfrom_path': path.get('copyfrom-path'),
 133                 'copyfrom_revision': copyfrom_rev,
 134             })
 135         # Sort paths (i.e. into hierarchical order), so that process_svn_log_entry()
 136         # can process actions in depth-first order.
 137         d['changed_paths'] = sorted(paths, key=operator.itemgetter('path'))
 138         revprops = []
 139         for prop in entry.findall('.//revprops/property'):
 140             revprops.append({ 'name': prop.get('name'), 'value': prop.text })
 141         d['revprops'] = revprops
 142         l.append(d)
 143     return l
 144
 145 def parse_svn_status_xml(xml_string, base_dir=None, ignore_externals=False):
 146     """
 147     Parse the XML output from an "svn status" command and extract useful info
 148     as a list of dicts (one per status entry).
 149     """
 150     if base_dir:
 151         base_dir = os.path.normcase(base_dir)
 152     l = []
 153     xml_string = strip_forbidden_xml_chars(xml_string)
 154     tree = ET.fromstring(xml_string)
 155     for entry in tree.findall('.//entry'):
 156         d = {}
 157         path = entry.get('path')
 158         if base_dir is not None and os.path.normcase(path).startswith(base_dir):
 159             path = path[len(base_dir):].lstrip('/\\')
 160         d['path'] = path
 161         wc_status = entry.find('wc-status')
 162         if wc_status.get('item') == 'external':
 163             if ignore_externals:
 164                 continue
 165         status =   wc_status.get('item')
 166         revision = wc_status.get('revision')
 167         if status == 'external':
 168             d['type'] = 'external'
 169         elif revision is not None:
 170             d['type'] = 'normal'
 171         else:
 172             d['type'] = 'unversioned'
 173         d['status'] =   status
 174         d['revision'] = revision
 175         d['props'] =    wc_status.get('props')
 176         d['copied'] =   wc_status.get('copied')
 177         l.append(d)
 178     return l
 179
 180 def get_svn_rev(svn_url_or_wc, rev_number):
 181     """
 182     Evaluate a given SVN revision pattern, to map it to a discrete rev #.
 183     """
 184     xml_string = run_svn(['info', '--xml', '-r', rev_number, svn_url_or_wc], fail_if_stderr=True)
 185     info = parse_svn_info_xml(xml_string)
 186     return info['revision']
 187
 188 def get_svn_info(svn_url_or_wc, rev_number=None):
 189     """
 190     Get SVN information for the given URL or working copy, with an optionally
 191     specified revision number.
 192     Returns a dict as created by parse_svn_info_xml().
 193     """
 194     args = ['info', '--xml']
 195     if rev_number is not None:
 196         args += ["-r", rev_number, svn_url_or_wc+"@"+str(rev_number)]
 197     else:
 198         args += [svn_url_or_wc]
 199     xml_string = run_svn(args, fail_if_stderr=True)
 200     return parse_svn_info_xml(xml_string)
 201
 202 def svn_checkout(svn_url, checkout_dir, rev_number=None):
 203     """
 204     Checkout the given URL at an optional revision number.
 205     """
 206     args = ['checkout', '-q']
 207     if rev_number is not None:
 208         args += ['-r', rev_number]
 209     args += [svn_url, checkout_dir]
 210     return run_svn(args)
 211
 212 def run_svn_log(svn_url_or_wc, rev_start, rev_end, limit, stop_on_copy=False, get_changed_paths=True, get_revprops=False):
 213     """
 214     Fetch up to 'limit' SVN log entries between the given revisions.
 215     """
 216     args = ['log', '--xml']
 217     if stop_on_copy:
 218         args += ['--stop-on-copy']
 219     if get_changed_paths:
 220         args += ['-v']
 221     if get_revprops:
 222         args += ['--with-all-revprops']
 223     url = str(svn_url_or_wc)
 224     args += ['-r', '%s:%s' % (rev_start, rev_end)]
 225     if not "@" in svn_url_or_wc:
 226         url = "%s@%s" % (svn_url_or_wc, str(max(rev_start, rev_end)))
 227     args += ['--limit', str(limit), url]
 228     xml_string = run_svn(args)
 229     return parse_svn_log_xml(xml_string)
 230
 231 def get_svn_status(svn_wc, quiet=False, no_recursive=False):
 232     """
 233     Get SVN status information about the given working copy.
 234     """
 235     # Ensure proper stripping by canonicalizing the path
 236     svn_wc = os.path.abspath(svn_wc)
 237     args = ['status', '--xml', '--ignore-externals']
 238     if quiet:
 239         args += ['-q']
 240     else:
 241         args += ['-v']
 242     if no_recursive:
 243         args += ['-N']
 244     xml_string = run_svn(args + [svn_wc])
 245     return parse_svn_status_xml(xml_string, svn_wc, ignore_externals=True)
 246
 247 def get_svn_versioned_files(svn_wc):
 248     """
 249     Get the list of versioned files in the SVN working copy.
 250     """
 251     contents = []
 252     for e in get_svn_status(svn_wc):
 253         if e['path'] and e['type'] == 'normal':
 254             contents.append(e['path'])
 255     return contents
 256
 257 def get_one_svn_log_entry(svn_url, rev_start, rev_end, stop_on_copy=False, get_changed_paths=True, get_revprops=False):
 258     """
 259     Get the first SVN log entry in the requested revision range.
 260     """
 261     entries = run_svn_log(svn_url, rev_start, rev_end, 1, stop_on_copy, get_changed_paths, get_revprops)
 262     if entries:
 263         return entries[0]
 264     raise EmptySVNLog("No SVN log for %s between revisions %s and %s" %
 265         (svn_url, rev_start, rev_end))
 266
 267 def get_first_svn_log_entry(svn_url, rev_start, rev_end, stop_on_copy=True, get_changed_paths=True):
 268     """
 269     Get the first log entry after (or at) the given revision number in an SVN branch.
 270     By default the revision number is set to 0, which will give you the log
 271     entry corresponding to the branch creaction.
 272
 273     NOTE: to know whether the branch creation corresponds to an SVN import or
 274     a copy from another branch, inspect elements of the 'changed_paths' entry
 275     in the returned dictionary.
 276     """
 277     return get_one_svn_log_entry(svn_url, rev_start, rev_end, stop_on_copy=stop_on_copy, get_changed_paths=get_changed_paths)
 278
 279 def get_last_svn_log_entry(svn_url, rev_start, rev_end, stop_on_copy=True, get_changed_paths=True):
 280     """
 281     Get the last log entry before/at the given revision number in an SVN branch.
 282     By default the revision number is set to HEAD, which will give you the log
 283     entry corresponding to the latest commit in branch.
 284     """
 285     return get_one_svn_log_entry(svn_url, rev_end, rev_start, stop_on_copy=stop_on_copy, get_changed_paths=get_changed_paths)
 286
 287
 288 log_duration_threshold = 10.0
 289 log_min_chunk_length = 10
 290 log_max_chunk_length = 10000
 291
 292 def iter_svn_log_entries(svn_url, first_rev, last_rev, stop_on_copy=False, get_changed_paths=True, get_revprops=False, ancestors=[]):
 293     """
 294     Iterate over SVN log entries between first_rev and last_rev.
 295
 296     This function features chunked log fetching so that it isn't too nasty
 297     to the SVN server if many entries are requested.
 298
 299     NOTE: If *not* passing in the explicit (pre-calculated) 'ancestors' list,
 300     this chunked log fetching *ONLY* works correctly on paths which
 301     are known to have existed unbroken in the SVN repository, e.g. /trunk.
 302     Chunked fetching breaks down if a path existed in earlier, then was
 303     deleted, and later was re-created. For example, if path was created in r5,
 304     then deleted in r1000, and then later re-created in r5000...
 305       svn log --stop-on-copy --limit 1 -r 1:50 "path/to/file"
 306         --> would yield r5, i.e. the _initial_ creation
 307       svn log --stop-on-copy --limit 1 -r 1:HEAD "path/to/file"
 308         --> would yield r5000, i.e. the _re-creation_
 309     Use run/svn2svn.py:find_svn_ancestors() to pass in the 'ancestors' array
 310     so that we can correctly re-trace ancestry here.
 311     """
 312     info = get_svn_info(svn_url)
 313     svn_repos_url = info['repos_url']
 314     #print "iter_svn_log_entries: %s %s:%s" % (svn_url, first_rev, last_rev)
 315     if last_rev == "HEAD":
 316         last_rev = info['revision']
 317     if int(first_rev) == 1:
 318         start_log = get_first_svn_log_entry(svn_url, first_rev, last_rev, stop_on_copy=stop_on_copy, get_changed_paths=False)
 319         if start_log['revision'] > first_rev:
 320             first_rev = start_log['revision']
 321     #print "first_rev: %s" % first_rev
 322     cur_url = svn_url
 323     cur_rev = first_rev
 324     cur_anc_idx = None
 325     cur_anc_end_rev = None
 326     if ancestors:
 327         #print ancestors
 328         # Crawl ancestry, from oldest to newest
 329         for idx in range(len(ancestors)-1, -1, -1):  # [n-1,...,0]
 330             #print "(pre) Match ancestors[%s]: %s" % (idx, ancestors[idx])
 331             cur_url = svn_repos_url+ancestors[idx]['copyfrom_path']
 332             cur_anc_idx = idx
 333             if first_rev < int(ancestors[idx]['copyfrom_rev']):
 334                 cur_anc_end_rev = int(ancestors[idx]['copyfrom_rev'])
 335                 break
 336         if cur_anc_end_rev is None:
 337             #print "(pre) Match ancestors[0] (final): %s" % (ancestors[0])
 338             cur_anc_idx = -1
 339             cur_url = svn_repos_url+ancestors[0]['path']
 340     chunk_length = log_min_chunk_length
 341     while cur_rev <= last_rev:
 342         #print "cur_rev:%s cur_anc_end_rev:%s cur_anc_idx:%s  %s" % (cur_rev, str(cur_anc_end_rev), cur_anc_idx, cur_url)
 343         if cur_anc_end_rev and cur_rev >= cur_anc_end_rev:
 344             cur_rev = int(ancestors[cur_anc_idx]['revision'])
 345             cur_anc_idx -= 1
 346             if cur_anc_idx >= 0:
 347                 idx = cur_anc_idx
 348                 #print "(loop) Match ancestors[%s]: %s" % (idx, ancestors[idx])
 349                 cur_url = svn_repos_url+ancestors[idx]['copyfrom_path']
 350                 cur_anc_end_rev = int(ancestors[idx]['copyfrom_rev'])
 351             else:
 352                 #print "(loop) Match ancestors[0] (final): %s" % (ancestors[0])
 353                 cur_url = svn_repos_url+ancestors[0]['path']
 354                 cur_anc_end_rev = None
 355         #print "cur_rev:%s cur_anc_end_rev:%s cur_anc_idx:%s  %s" % (cur_rev, str(cur_anc_end_rev), cur_anc_idx, cur_url)
 356         start_t = time.time()
 357         stop_rev = min(last_rev, cur_rev + chunk_length)
 358         stop_rev = min(stop_rev, cur_anc_end_rev) if cur_anc_end_rev else stop_rev
 359         entries = run_svn_log(cur_url, cur_rev, stop_rev, chunk_length,
 360                               stop_on_copy, get_changed_paths, get_revprops)
 361         duration = time.time() - start_t
 362         if entries:
 363             for e in entries:
 364                 if e['revision'] > last_rev:
 365                     break
 366                 # Embed the current URL in the yielded dict, for ancestor cases where
 367                 # we might have followed a copy-from to some non-original URL.
 368                 e['url'] = cur_url
 369                 yield e
 370             if e['revision'] >= last_rev:
 371                 break
 372             cur_rev = int(e['revision'])+1
 373         else:
 374             cur_rev = int(stop_rev)+1
 375         # Adapt chunk length based on measured request duration
 376         if duration < log_duration_threshold:
 377             chunk_length = min(log_max_chunk_length, int(chunk_length * 2.0))
 378         elif duration > log_duration_threshold * 2:
 379             chunk_length = max(log_min_chunk_length, int(chunk_length / 2.0))
 380
 381
 382 _svn_client_version = None
 383
 384 def get_svn_client_version():
 385     """
 386     Returns the SVN client version as a tuple.
 387
 388     The returned tuple only contains numbers, non-digits in version string are
 389     silently ignored.
 390     """
 391     global _svn_client_version
 392     if _svn_client_version is None:
 393         raw = run_svn(['--version', '-q']).strip()
 394         _svn_client_version = tuple(map(int, [x for x in raw.split('.')
 395                                               if x.isdigit()]))
 396     return _svn_client_version
 397
 398
 399 def parse_svn_propget_xml(xml_string):
 400     """
 401     Parse the XML output from an "svn propget" command and extract useful
 402     information as a dict.
 403     """
 404     d = {}
 405     xml_string = strip_forbidden_xml_chars(xml_string)
 406     tree = ET.fromstring(xml_string)
 407     prop = tree.find('.//property')
 408     d['name'] = prop.get('name')
 409     d['value'] = prop is not None and prop.text and prop.text.replace('\r\n', '\n').replace('\n\r', '\n').replace('\r', '\n') or ""
 410     return d
 411
 412 def parse_svn_proplist_xml(xml_string):
 413     """
 414     Parse the XML output from an "svn proplist" command and extract list
 415     of property-names.
 416     """
 417     l = []
 418     xml_string = strip_forbidden_xml_chars(xml_string)
 419     tree = ET.fromstring(xml_string)
 420     for prop in tree.findall('.//property'):
 421         l.append(prop.get('name'))
 422     return l
 423
 424 def get_prop_value(svn_url_or_wc, prop_name, rev_number=None):
 425     """
 426     Get the value of a versioned property for the given path.
 427     """
 428     args = ['propget', '--xml']
 429     url = str(svn_url_or_wc)
 430     if rev_number:
 431         args += ['-r', rev_number]
 432         if not "@" in svn_url_or_wc:
 433             url = "%s@%s" % (svn_url_or_wc, str(rev_number))
 434     args += [prop_name, url]
 435     xml_string = run_svn(args)
 436     return parse_svn_propget_xml(xml_string)
 437
 438 def get_all_props(svn_url_or_wc, rev_number=None):
 439     """
 440     Get the values of all versioned properties for the given path.
 441     """
 442     l = {}
 443     args = ['proplist', '--xml']
 444     url = str(svn_url_or_wc)
 445     if rev_number:
 446         args += ['-r', rev_number]
 447         if not "@" in svn_url_or_wc:
 448             url = "%s@%s" % (svn_url_or_wc, str(rev_number))
 449     args += [url]
 450     xml_string = run_svn(args)
 451     props = parse_svn_proplist_xml(xml_string)
 452     for prop_name in props:
 453         d = get_prop_value(svn_url_or_wc, prop_name, rev_number)
 454         l[d['name']] = d['value']
 455     return l