svn2svn/run/svn2svn.py

   1 """
   2 Replicate (replay) changesets from one SVN repository to another.
   3 """
   4
   5 from .. import base_version, full_version
   6 from .. import ui
   7 from .. import svnclient
   8 from ..shell import run_svn
   9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
  10 from parse import HelpFormatter
  11
  12 import sys
  13 import os
  14 import time
  15 import traceback
  16 import shutil
  17 import operator
  18 import optparse
  19 from datetime import datetime
  20
  21 _valid_svn_actions = "MARD"   # The list of known SVN action abbr's, from "svn log"
  22
  23 # Module-level variables/parameters
  24 source_url = ""          # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
  25 source_repos_url = ""    # URL to root of source SVN repo,        e.g. 'http://server/svn/source'
  26 source_base = ""         # Relative path of source_url in source SVN repo, e.g. '/trunk'
  27 source_repos_uuid = ""   # UUID of source SVN repo
  28 target_url =""           # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
  29 rev_map = {}             # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
  30
  31 def commit_from_svn_log_entry(log_entry, options, commit_paths=None, target_revprops=None):
  32     """
  33     Given an SVN log entry and an optional list of changed paths, do an svn commit.
  34     """
  35     # TODO: Run optional external shell hook here, for doing pre-commit filtering
  36     # Display the _wc_target "svn status" info if running in -vv (or higher) mode
  37     if ui.get_level() >= ui.EXTRA:
  38         ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
  39         ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
  40     # This will use the local timezone for displaying commit times
  41     timestamp = int(log_entry['date'])
  42     svn_date = str(datetime.fromtimestamp(timestamp))
  43     # Uncomment this one one if you prefer UTC commit times
  44     #svn_date = "%d 0" % timestamp
  45     args = ["commit", "--force-log"]
  46     if options.keep_author:
  47         args += ["-m", log_entry['message'] + "\nDate: " + svn_date, "--username", log_entry['author']]
  48     else:
  49         args += ["-m", log_entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + log_entry['author']]
  50     revprops = {}
  51     if log_entry['revprops']:
  52         # Carry forward any revprop's from the source revision
  53         for v in log_entry['revprops']:
  54             revprops[v['name']] = v['value']
  55     if target_revprops:
  56         # Add any extra revprop's we want to set for the target repo commits
  57         for v in target_revprops:
  58             revprops[v['name']] = v['value']
  59     if revprops:
  60         for key in revprops:
  61             args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
  62     if commit_paths:
  63         if len(commit_paths)<100:
  64             # If we don't have an excessive amount of individual changed paths, pass
  65             # those to the "svn commit" command. Else, pass nothing so we commit at
  66             # the root of the working-copy.
  67             args += list(commit_paths)
  68     rev = None
  69     if not options.dry_run:
  70         # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
  71         output = run_svn(args)
  72         if output:
  73             output_lines = output.strip("\n").split("\n")
  74             rev = ""
  75             for line in output_lines:
  76                 if line[0:19] == 'Committed revision ':
  77                     rev = line[19:].rstrip('.')
  78                     break
  79             if rev:
  80                 ui.status("Committed revision %s.", rev)
  81     return rev
  82
  83 def full_svn_revert():
  84     """
  85     Do an "svn revert" and proactively remove any extra files in the working copy.
  86     """
  87     run_svn(["revert", "--recursive", "."])
  88     output = run_svn(["status"])
  89     if output:
  90         output_lines = output.strip("\n").split("\n")
  91         for line in output_lines:
  92             if line[0] == "?":
  93                 path = line[4:].strip(" ")
  94                 if os.path.isfile(path):
  95                     os.remove(path)
  96                 if os.path.isdir(path):
  97                     shutil.rmtree(path)
  98
  99 def gen_tracking_revprops(source_rev):
 100     """
 101     Build an array of svn2svn-specific source-tracking revprops.
 102     """
 103     revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
 104                 {'name':'svn2svn:source_url',  'value':source_url},
 105                 {'name':'svn2svn:source_rev',  'value':source_rev}]
 106     return revprops
 107
 108 def sync_svn_props(source_url, source_rev, path_offset):
 109     """
 110     Carry-forward any unversioned properties from the source repo to the
 111     target WC.
 112     """
 113     source_props = svnclient.get_all_props(source_url+"/"+path_offset,  source_rev)
 114     target_props = svnclient.get_all_props(path_offset)
 115     if 'svn:mergeinfo' in source_props:
 116         # Never carry-forward "svn:mergeinfo"
 117         del source_props['svn:mergeinfo']
 118     for prop in target_props:
 119         if prop not in source_props:
 120             # Remove any properties which exist in target but not source
 121             run_svn(["propdel", prop, path_offset])
 122     for prop in source_props:
 123         if prop not in target_props or \
 124            source_props[prop] != target_props[prop]:
 125             # Set/update any properties which exist in source but not target or
 126             # whose value differs between source vs. target.
 127             run_svn(["propset", prop, source_props[prop], path_offset])
 128
 129 def in_svn(p, require_in_repo=False, prefix=""):
 130     """
 131     Check if a given file/folder is being tracked by Subversion.
 132     Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
 133     With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
 134     Use "svn status" to check the status of the file/folder.
 135     """
 136     entries = svnclient.get_svn_status(p, no_recursive=True)
 137     if not entries:
 138         return False
 139     d = entries[0]
 140     if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
 141         # If caller requires this path to be in the SVN repo, prevent returning True
 142         # for paths that are only locally-added.
 143         ret = False
 144     else:
 145         # Don't consider files tracked as deleted in the WC as under source-control.
 146         # Consider files which are locally added/copied as under source-control.
 147         ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
 148     ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
 149     return ret
 150
 151 def is_child_path(path, p_path):
 152     return True if (path == p_path) or (path.startswith(p_path+"/")) else False
 153
 154 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
 155     """
 156     Given a source path, walk the SVN history backwards to inspect the ancestory of
 157     that path, seeing if it traces back to base_path.  Build an array of copyfrom_path
 158     and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
 159     which base_path is a substring match of (e.g. we crawled back to the initial branch-
 160     copy from trunk), then return the collection of ancestor paths.  Otherwise,
 161     copyfrom_path has no ancestory compared to base_path.
 162
 163     This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
 164     file/folder was renamed in a branch and then that branch was merged back to trunk.
 165
 166     'svn_repos_url' is the full URL to the root of the SVN repository,
 167       e.g. 'file:///path/to/repo'
 168     'base_path' is the path in the SVN repo to the target path we're trying to
 169       trace ancestry back to, e.g. 'trunk'.
 170     'source_path' is the path in the SVN repo to the source path to start checking
 171       ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
 172       (full_path = svn_repos_url+base_path+"/"+path_offset)
 173     'source_rev' is the revision to start walking the history of source_path backwards from.
 174     """
 175     ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s  base_path: %s",
 176         svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
 177     done = False
 178     working_path = base_path+"/"+source_path
 179     working_rev  = source_rev
 180     first_iter_done = False
 181     ancestors_temp = []
 182     while not done:
 183         # Get the first "svn log" entry for this path (relative to @rev)
 184         ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
 185         log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path, 1, working_rev, True)
 186         if not log_entry:
 187             ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
 188             done = True
 189             break
 190         # If we found a copy-from case which matches our base_path, we're done.
 191         # ...but only if we've at least tried to search for the first copy-from path.
 192         if first_iter_done and is_child_path(working_path, base_path):
 193             ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(working_path, base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
 194             done = True
 195             break
 196         first_iter_done = True
 197         # Search for any actions on our target path (or parent paths).
 198         changed_paths_temp = []
 199         for d in log_entry['changed_paths']:
 200             path = d['path']
 201             if path in working_path:
 202                 changed_paths_temp.append({'path': path, 'data': d})
 203         if not changed_paths_temp:
 204             # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
 205             ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
 206             done = True
 207             continue
 208         # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
 209         changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
 210         # Find the action for our working_path in this revision. Use a loop to check in reverse order,
 211         # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
 212         for v in changed_paths:
 213             d = v['data']
 214             path = d['path']
 215             # Check action-type for this file
 216             action = d['action']
 217             if action not in _valid_svn_actions:
 218                 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 219                     % (log_entry['revision'], action))
 220             ui.status(prefix + "> %s %s%s", action, path,
 221                 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 222                 level=ui.DEBUG, color='YELLOW')
 223             if action == 'D':
 224                 # If file/folder was deleted, it has no ancestor
 225                 ancestors_temp = []
 226                 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
 227                 done = True
 228                 break
 229             if action in 'RA':
 230                 # If file/folder was added/replaced but not a copy, it has no ancestor
 231                 if not d['copyfrom_path']:
 232                     ancestors_temp = []
 233                     ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
 234                         "Added" if action == "A" else "Replaced",
 235                         level=ui.DEBUG, color='YELLOW')
 236                     done = True
 237                     break
 238                 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
 239                 # and keep checking for ancestors
 240                 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
 241                     action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
 242                     level=ui.DEBUG, color='YELLOW')
 243                 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
 244                                        'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
 245                 working_path = working_path.replace(d['path'], d['copyfrom_path'])
 246                 working_rev =  d['copyfrom_revision']
 247                 # Follow the copy and keep on searching
 248                 break
 249     ancestors = []
 250     if ancestors_temp:
 251         ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
 252         working_path = base_path+"/"+source_path
 253         for idx in range(len(ancestors_temp)):
 254             d = ancestors_temp[idx]
 255             working_path = working_path.replace(d['path'], d['copyfrom_path'])
 256             working_rev =  d['copyfrom_rev']
 257             ancestors.append({'path': working_path, 'revision': working_rev})
 258         if ui.get_level() >= ui.DEBUG:
 259             max_len = 0
 260             for idx in range(len(ancestors)):
 261                 d = ancestors[idx]
 262                 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
 263             ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
 264             for idx in range(len(ancestors)-1):
 265                 d = ancestors[idx]
 266                 d_next = ancestors[idx+1]
 267                 ui.status(prefix + " [%s] %s <-- %s", idx,
 268                     str(d['path']+"@"+str(d['revision'])).ljust(max_len),
 269                     str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
 270                     level=ui.DEBUG, color='YELLOW')
 271     else:
 272         ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
 273             svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
 274     return ancestors
 275
 276 def get_rev_map(source_rev, prefix):
 277     """
 278     Find the equivalent rev # in the target repo for the given rev # from the source repo.
 279     """
 280     ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
 281     # Find the highest entry less-than-or-equal-to source_rev
 282     for rev in range(int(source_rev), 0, -1):
 283         ui.status(prefix + ">> get_rev_map: rev=%s  in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
 284         if rev in rev_map:
 285             return int(rev_map[rev])
 286     # Else, we fell off the bottom of the rev_map. Ruh-roh...
 287     return None
 288
 289 def set_rev_map(source_rev, target_rev):
 290     ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
 291     global rev_map
 292     rev_map[int(source_rev)]=int(target_rev)
 293
 294 def build_rev_map(target_url, target_end_rev, source_info):
 295     """
 296     Check for any already-replayed history from source_url (source_info) and
 297     build the mapping-table of source_rev -> target_rev.
 298     """
 299     global rev_map
 300     rev_map = {}
 301     ui.status("Rebuilding rev_map...", level=ui.VERBOSE)
 302     proc_count = 0
 303     it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
 304     for log_entry in it_log_entries:
 305         if log_entry['revprops']:
 306             revprops = {}
 307             for v in log_entry['revprops']:
 308                 if v['name'].startswith('svn2svn:'):
 309                     revprops[v['name']] = v['value']
 310             if revprops and \
 311                revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
 312                revprops['svn2svn:source_url'] == source_info['url']:
 313                 source_rev = revprops['svn2svn:source_rev']
 314                 target_rev = log_entry['revision']
 315                 set_rev_map(source_rev, target_rev)
 316
 317 def get_svn_dirlist(svn_path, svn_rev = ""):
 318     """
 319     Get a list of all the child contents (recusive) of the given folder path.
 320     """
 321     args = ["list"]
 322     path = svn_path
 323     if svn_rev:
 324         args += ["-r", svn_rev]
 325         path += "@"+str(svn_rev)
 326     args += [path]
 327     paths = run_svn(args, no_fail=True)
 328     paths = paths.strip("\n").split("\n") if len(paths)>1 else []
 329     return paths
 330
 331 def path_in_list(paths, path):
 332     for p in paths:
 333         if is_child_path(path, p):
 334             return True
 335     return False
 336
 337 def add_path(paths, path):
 338     if not path_in_list(paths, path):
 339         paths.append(path)
 340
 341 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
 342                export_paths={}, is_dir = False, skip_paths=[], prefix = ""):
 343     """
 344     Given the add'd source path, replay the "svn add/copy" commands to correctly
 345     track renames across copy-from's.
 346
 347     For example, consider a sequence of events like this:
 348     1. svn copy /trunk /branches/fix1
 349     2. (Make some changes on /branches/fix1)
 350     3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2  " Rename folder
 351     4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt  " Rename file inside renamed folder
 352     5. svn co /trunk && svn merge /branches/fix1
 353     After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
 354     and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
 355     to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
 356     we'd lose the logical history that Proj2/file2.txt is really a descendant
 357     of Proj1/file1.txt.
 358
 359     'path_offset' is the offset from source_base to the file to check ancestry for,
 360       e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
 361     'source_rev' is the revision ("svn log") that we're processing from the source repo.
 362     'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
 363       directory, when being called recursively by do_svn_add_dir().
 364     'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
 365     'is_dir' is whether path_offset is a directory (rather than a file).
 366     """
 367     ui.status(prefix + ">> do_svn_add: %s  %s", source_base+"/"+path_offset+"@"+str(source_rev),
 368         "  (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
 369         level=ui.DEBUG, color='GREEN')
 370     # Check if the given path has ancestors which chain back to the current source_base
 371     found_ancestor = False
 372     ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+"  ")
 373     # ancestors[n] is the original (pre-branch-copy) trunk path.
 374     # ancestors[n-1] is the first commit on the new branch.
 375     copyfrom_path = ancestors[len(ancestors)-1]['path']     if ancestors else ""
 376     copyfrom_rev  = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
 377     if ancestors:
 378         # The copy-from path has ancestory back to source_url.
 379         ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
 380             level=ui.DEBUG, color='GREEN', bold=True)
 381         found_ancestor = True
 382         # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
 383         # return None in the case where copyfrom_rev is *before* our source_start_rev.
 384         tgt_rev = get_rev_map(copyfrom_rev, prefix+"  ")
 385         ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
 386     else:
 387         ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
 388         found_ancestor = False
 389     if found_ancestor and tgt_rev:
 390         # Check if this path_offset in the target WC already has this ancestry, in which
 391         # case there's no need to run the "svn copy" (again).
 392         path_in_svn = in_svn(path_offset, prefix=prefix+"  ")
 393         log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+"  ") else []
 394         if (not log_entry or (log_entry['revision'] != tgt_rev)):
 395             copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
 396             ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 397             ui.status(prefix + "   copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 398             ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
 399             if path_in_svn and \
 400                ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
 401                 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
 402                 # When being called recursively, if this child entry has the same ancestor as the
 403                 # the parent, then no need to try to run another "svn copy".
 404                 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
 405                     parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
 406                 pass
 407             else:
 408                 # Copy this path from the equivalent path+rev in the target repo, to create the
 409                 # equivalent history.
 410                 if parent_copyfrom_path:
 411                     # If we have a parent copy-from path, we mis-match that so display a status
 412                     # message describing the action we're mimic'ing. If path_in_svn, then this
 413                     # is logically a "replace" rather than an "add".
 414                     ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
 415                 if path_in_svn:
 416                     # If local file is already under version-control, then this is a replace.
 417                     ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
 418                     run_svn(["remove", "--force", path_offset])
 419                 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
 420                 if is_dir:
 421                     # Export the final verison of all files in this folder.
 422                     add_path(export_paths, path_offset)
 423                 else:
 424                     # Export the final verison of this file.
 425                     run_svn(["export", "--force", "-r", source_rev,
 426                              source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 427                 # Copy SVN properties from source repo
 428                 sync_svn_props(source_url, source_rev, path_offset)
 429         else:
 430             ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
 431     else:
 432         # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
 433         # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
 434         # Create (parent) directory if needed.
 435         # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
 436         #       split-out to a shared tag?
 437         p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
 438         if not os.path.exists(p_path):
 439             run_svn(["mkdir", p_path])
 440         if not in_svn(path_offset, prefix=prefix+"  "):
 441             if is_dir:
 442                 # Export the final verison of all files in this folder.
 443                 add_path(export_paths, path_offset)
 444             else:
 445                 # Export the final verison of this file. We *need* to do this before running
 446                 # the "svn add", even if we end-up re-exporting this file again via export_paths.
 447                 run_svn(["export", "--force", "-r", source_rev,
 448                          source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 449             # If not already under version-control, then "svn add" this file/folder.
 450             run_svn(["add", "--parents", path_offset])
 451         # Copy SVN properties from source repo
 452         sync_svn_props(source_url, source_rev, path_offset)
 453     if is_dir:
 454         # For any folders that we process, process any child contents, so that we correctly
 455         # replay copies/replaces/etc.
 456         do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, skip_paths, prefix+"  ")
 457
 458 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
 459                    export_paths, skip_paths, prefix=""):
 460     # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
 461     # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
 462     #       associated remote repo folder. (Is this a problem?)
 463     paths_local =  get_svn_dirlist(path_offset)
 464     paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
 465     ui.status(prefix + ">> do_svn_add_dir: paths_local:  %s", str(paths_local),  level=ui.DEBUG, color='GREEN')
 466     ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
 467     # Update files/folders which exist in remote but not local
 468     for path in paths_remote:
 469         path_is_dir = True if path[-1] == "/" else False
 470         working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
 471         if not working_path in skip_paths:
 472             do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
 473                        export_paths, path_is_dir, skip_paths, prefix+"  ")
 474     # Remove files/folders which exist in local but not remote
 475     for path in paths_local:
 476         if not path in paths_remote:
 477             ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
 478             run_svn(["remove", "--force", path_offset+"/"+path])
 479             # TODO: Does this handle deleted folders too? Wouldn't want to have a case
 480             #       where we only delete all files from folder but leave orphaned folder around.
 481
 482 def process_svn_log_entry(log_entry, options, commit_paths, prefix = ""):
 483     """
 484     Process SVN changes from the given log entry. Build an array (commit_paths)
 485     of the paths in the working-copy that were changed, i.e. the paths which
 486     we'll pass to "svn commit".
 487     """
 488     export_paths = []
 489     source_rev = log_entry['revision']
 490     ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
 491     for d in log_entry['changed_paths']:
 492         # Get the full path for this changed_path
 493         # e.g. '/branches/bug123/projectA/file1.txt'
 494         path = d['path']
 495         if not is_child_path(path, source_base):
 496             # Ignore changed files that are not part of this subdir
 497             ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s  (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
 498             continue
 499         # Note: d['kind']="" for action="M" paths which only have property changes.
 500         if d['kind'] == "":
 501             d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
 502         assert (d['kind'] == 'file') or (d['kind'] == 'dir')
 503         path_is_dir =  True if d['kind'] == 'dir'  else False
 504         path_is_file = True if d['kind'] == 'file' else False
 505         # Calculate the offset (based on source_base) for this changed_path
 506         # e.g. 'projectA/file1.txt'
 507         # (path = source_base + "/" + path_offset)
 508         path_offset = path[len(source_base):].strip("/")
 509         # Get the action for this path
 510         action = d['action']
 511         if action not in _valid_svn_actions:
 512             raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 513                 % (source_rev, action))
 514         ui.status(" %s %s%s", action, d['path'],
 515             (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 516             level=ui.VERBOSE)
 517
 518         # Try to be efficient and keep track of an explicit list of paths in the
 519         # working copy that changed. If we commit from the root of the working copy,
 520         # then SVN needs to crawl the entire working copy looking for pending changes.
 521         commit_paths.append(path_offset)
 522
 523         # Special-handling for replace's
 524         if action == 'R':
 525             # If file was "replaced" (deleted then re-added, all in same revision),
 526             # then we need to run the "svn rm" first, then change action='A'. This
 527             # lets the normal code below handle re-"svn add"'ing the files. This
 528             # should replicate the "replace".
 529             if in_svn(path_offset):
 530                 # Target path might not be under version-control yet, e.g. parent "add"
 531                 # was a copy-from a branch which had no ancestry back to trunk, and each
 532                 # child folder under that parent folder is a "replace" action on the final
 533                 # merge to trunk. Since the child folders will be in skip_paths, do_svn_add
 534                 # wouldn't have created them while processing the parent "add" path.
 535                 run_svn(["remove", "--force", path_offset])
 536             action = 'A'
 537
 538         # Handle all the various action-types
 539         # (Handle "add" first, for "svn copy/move" support)
 540         if action == 'A':
 541             # Determine where to export from.
 542             svn_copy = False
 543             # Handle cases where this "add" was a copy from another URL in the source repo
 544             if d['copyfrom_revision']:
 545                 copyfrom_path = d['copyfrom_path']
 546                 copyfrom_rev =  d['copyfrom_revision']
 547                 skip_paths = []
 548                 for tmp_d in log_entry['changed_paths']:
 549                     tmp_path = tmp_d['path']
 550                     if is_child_path(tmp_path, path):
 551                         # Build list of child entries which are also in the changed_paths list,
 552                         # so that do_svn_add() can skip processing these entries when recursing
 553                         # since we'll end-up processing them later.
 554                         tmp_path_offset = tmp_path[len(source_base):].strip("/")
 555                         skip_paths.append(tmp_path_offset)
 556                 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, skip_paths, prefix+"  ")
 557             # Else just "svn export" the files from the source repo and "svn add" them.
 558             else:
 559                 # Create (parent) directory if needed
 560                 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
 561                 if not os.path.exists(p_path):
 562                     run_svn(["mkdir", p_path])
 563                 # Export the entire added tree.
 564                 if path_is_dir:
 565                     # For directories, defer the (recurisve) "svn export". Might have a
 566                     # situation in a branch merge where the entry in the svn-log is a
 567                     # non-copy-from'd "add" but there are child contents (that we haven't
 568                     # gotten to yet in log_entry) that are copy-from's.  When we try do
 569                     # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
 570                     # having pre-existing (svn-add'd) contents creates some trouble.
 571                     # Instead, just create the stub folders ("svn mkdir" above) and defer
 572                     # exporting the final file-state until the end.
 573                     add_path(export_paths, path_offset)
 574                 else:
 575                     # Export the final verison of this file. We *need* to do this before running
 576                     # the "svn add", even if we end-up re-exporting this file again via export_paths.
 577                     run_svn(["export", "--force", "-r", source_rev,
 578                              source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 579                 if not in_svn(path_offset, prefix=prefix+"  "):
 580                     # Need to use in_svn here to handle cases where client committed the parent
 581                     # folder and each indiv sub-folder.
 582                     run_svn(["add", "--parents", path_offset])
 583                 # Copy SVN properties from source repo
 584                 sync_svn_props(source_url, source_rev, path_offset)
 585
 586         elif action == 'D':
 587             run_svn(["remove", "--force", path_offset])
 588
 589         elif action == 'M':
 590             if path_is_file:
 591                 run_svn(["export", "--force", "-N" , "-r", source_rev,
 592                          source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 593             # Update SVN properties based on source repo
 594             sync_svn_props(source_url, source_rev, path_offset)
 595
 596         else:
 597             raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
 598                 % action)
 599
 600     # Export the final version of all add'd paths from source_url
 601     if export_paths:
 602         for path_offset in export_paths:
 603             run_svn(["export", "--force", "-r", source_rev,
 604                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 605
 606 def disp_svn_log_summary(log_entry):
 607     ui.status("")
 608     ui.status("r%s | %s | %s",
 609         log_entry['revision'],
 610         log_entry['author'],
 611         str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
 612     ui.status(log_entry['message'])
 613     ui.status("------------------------------------------------------------------------")
 614
 615 def real_main(options, args):
 616     global source_url, target_url, rev_map
 617     source_url = args.pop(0).rstrip("/")    # e.g. 'http://server/svn/source/trunk'
 618     target_url = args.pop(0).rstrip("/")    # e.g. 'file:///svn/target/trunk'
 619     ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
 620
 621     # Make sure that both the source and target URL's are valid
 622     source_info = svnclient.get_svn_info(source_url)
 623     assert is_child_path(source_url, source_info['repos_url'])
 624     target_info = svnclient.get_svn_info(target_url)
 625     assert is_child_path(target_url, target_info['repos_url'])
 626
 627     # Init global vars
 628     global source_repos_url,source_base,source_repos_uuid
 629     source_repos_url = source_info['repos_url']       # e.g. 'http://server/svn/source'
 630     source_base = source_url[len(source_repos_url):]  # e.g. '/trunk'
 631     source_repos_uuid = source_info['repos_uuid']
 632
 633     # Init start and end revision
 634     source_start_rev = svnclient.get_svn_rev(source_repos_url, options.svn_rev_start if options.svn_rev_start else 1)
 635     source_end_rev   = svnclient.get_svn_rev(source_repos_url, options.svn_rev_end   if options.svn_rev_end   else "HEAD")
 636
 637     target_end_rev = target_info['revision']   # Last revision # in the target repo
 638     wc_target = os.path.abspath('_wc_target')
 639     num_entries_proc = 0
 640     commit_count = 0
 641     source_rev = None
 642     target_rev = None
 643
 644     # Check out a working copy of target_url if needed
 645     wc_exists = os.path.exists(wc_target)
 646     if wc_exists and not options.cont_from_break:
 647         shutil.rmtree(wc_target)
 648         wc_exists = False
 649     if not wc_exists:
 650         ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
 651         svnclient.svn_checkout(target_url, wc_target)
 652     os.chdir(wc_target)
 653
 654     if not options.cont_from_break:
 655         # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
 656         # Get the first log entry at/after source_start_rev, which is where
 657         # we'll do the initial import from.
 658         it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False)
 659         for source_start_log in it_log_start:
 660             break
 661         if not source_start_log:
 662             raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
 663                 (source_start_rev, source_end_rev, source_url))
 664
 665         # This is the revision we will start from for source_url
 666         source_start_rev = source_rev = int(source_start_log['revision'])
 667         ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
 668
 669         # For the initial commit to the target URL, export all the contents from
 670         # the source URL at the start-revision.
 671         disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
 672         ui.status("(Initial import)", level=ui.VERBOSE)
 673         # Export and add file-contents from source_url@source_start_rev
 674         top_paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
 675         top_paths = top_paths.strip("\n").split("\n")
 676         for path in top_paths:
 677             # For each top-level file/folder...
 678             if not path:
 679                 continue
 680             # Directories have a trailing slash in the "svn list" output
 681             path_is_dir = True if path[-1] == "/" else False
 682             path_offset = path.rstrip('/') if path_is_dir else path
 683             if in_svn(path_offset, prefix="  "):
 684                 raise InternalError("Cannot replay history on top of pre-existing structure: %s" % source_url+"/"+path_offset)
 685             if path_is_dir and not os.path.exists(path_offset):
 686                 os.makedirs(path_offset)
 687             run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 688             run_svn(["add", path_offset])
 689         # Update any properties on the newly added content
 690         paths = run_svn(["list", "--recursive", "-r", source_rev, source_url+"@"+str(source_rev)])
 691         paths = paths.strip("\n").split("\n")
 692         sync_svn_props(source_url, source_rev, "")
 693         for path in paths:
 694             if not path:
 695                 continue
 696             # Directories have a trailing slash in the "svn list" output
 697             path_is_dir = True if path[-1] == "/" else False
 698             path_offset = path.rstrip('/') if path_is_dir else path
 699             ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path_offset, level=ui.VERBOSE)
 700             sync_svn_props(source_url, source_rev, path_offset)
 701         # Commit the initial import
 702         num_entries_proc += 1
 703         target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 704         target_rev = commit_from_svn_log_entry(source_start_log, options, target_revprops=target_revprops)
 705         if target_rev:
 706             # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 707             set_rev_map(source_rev, target_rev)
 708             # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 709             run_svn(["update"])
 710             commit_count += 1
 711     else:
 712         # Re-build the rev_map based on any already-replayed history in target_url
 713         build_rev_map(target_url, target_end_rev, source_info)
 714         if not rev_map:
 715             raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url)
 716         source_start_rev = int(max(rev_map, key=rev_map.get))
 717         assert source_start_rev
 718         ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
 719
 720     svn_vers_t = svnclient.get_svn_client_version()
 721     svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
 722
 723     # Load SVN log starting from source_start_rev + 1
 724     it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else []
 725     source_rev = None
 726
 727     try:
 728         for log_entry in it_log_entries:
 729             if options.entries_proc_limit:
 730                 if num_entries_proc >= options.entries_proc_limit:
 731                     break
 732             # Replay this revision from source_url into target_url
 733             disp_svn_log_summary(log_entry)
 734             source_rev = log_entry['revision']
 735             # Process all the changed-paths in this log entry
 736             commit_paths = []
 737             process_svn_log_entry(log_entry, options, commit_paths)
 738             num_entries_proc += 1
 739             # Commit any changes made to _wc_target
 740             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 741             target_rev = commit_from_svn_log_entry(log_entry, options, commit_paths, target_revprops=target_revprops)
 742             if target_rev:
 743                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 744                 source_rev = log_entry['revision']
 745                 set_rev_map(source_rev, target_rev)
 746                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 747                 run_svn(["update"])
 748                 commit_count += 1
 749                 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
 750                 if svn_vers >= 1.7 and (commit_count % 100 == 0):
 751                     run_svn(["cleanup"])
 752         if not source_rev:
 753             # If there were no new source_url revisions to process, init source_rev
 754             # for the "finally" message below.
 755             source_rev = source_end_rev
 756
 757     except KeyboardInterrupt:
 758         print "\nStopped by user."
 759         print "\nCleaning-up..."
 760         run_svn(["cleanup"])
 761         full_svn_revert()
 762     except:
 763         print "\nCommand failed with following error:\n"
 764         traceback.print_exc()
 765         print "\nCleaning-up..."
 766         run_svn(["cleanup"])
 767         print run_svn(["status"])
 768         full_svn_revert()
 769     finally:
 770         print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
 771
 772 def main():
 773     # Defined as entry point. Must be callable without arguments.
 774     usage = "Usage: %prog [OPTIONS] source_url target_url"
 775     description = """\
 776   Replicate (replay) history from one SVN repository to another. Maintain
 777   logical ancestry wherever possible, so that 'svn log' on the replayed
 778   repo will correctly follow file/folder renames.
 779
 780   == Examples ==
 781   Create a copy of only /trunk from source repo, starting at r5000
 782   $ svnadmin create /svn/target
 783   $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
 784   $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
 785     1. The target_url will be checked-out to ./_wc_target
 786     2. The first commit to http://server/source/trunk at/after r5000 will be
 787        exported & added into _wc_target
 788     3. All revisions affecting http://server/source/trunk (starting at r5000)
 789        will be replayed to _wc_target. Any add/copy/move/replaces that are
 790        copy-from'd some path outside of /trunk (e.g. files renamed on a
 791        /branch and branch was merged into /trunk) will correctly maintain
 792        logical ancestry where possible.
 793
 794   Use continue-mode (-c) to pick-up where the last run left-off
 795   $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
 796     1. The target_url will be checked-out to ./_wc_target, if not already
 797        checked-out
 798     2. All new revisions affecting http://server/source/trunk starting from
 799        the last replayed revision to file:///svn/target/trunk (based on the
 800        svn2svn:* revprops) will be replayed to _wc_target, maintaining all
 801        logical ancestry where possible."""
 802     parser = optparse.OptionParser(usage, description=description,
 803                 formatter=HelpFormatter(), version="%prog "+str(full_version))
 804     #parser.remove_option("--help")
 805     #parser.add_option("-h", "--help", dest="show_help", action="store_true",
 806     #    help="show this help message and exit")
 807     parser.add_option("-r", "--revision", type="string", dest="svn_rev", metavar="ARG",
 808                       help="revision range to replay from source_url\n" + \
 809                            "A revision argument can be one of:\n" + \
 810                            "   START        start rev # (end will be 'HEAD')\n" + \
 811                            "   START:END    start and ending rev #'s\n" + \
 812                            "(Any revision # formats which SVN understands\n" + \
 813                            " are supported, e.g. 'HEAD', '{2010-01-31}', etc.)")
 814     parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", default=False,
 815                       help="maintain original 'Author' info from source repo")
 816     parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
 817                       help="continue from previous break")
 818     parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
 819                       help="maximum number of log entries to process")
 820     parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 821                       help="try processing next log entry but don't commit changes to "
 822                            "target working-copy (forces --limit=1)")
 823     parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
 824                       help="enable additional output (use -vv or -vvv for more)")
 825     parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
 826                       help="enable debugging output (same as -vvv)")
 827     options, args = parser.parse_args()
 828     if len(args) != 2:
 829         parser.error("incorrect number of arguments")
 830     if options.verbosity < 10:
 831         # Expand multiple "-v" arguments to a real ui._level value
 832         options.verbosity *= 10
 833     if options.dry_run:
 834         # When in dry-run mode, only try to process the next log_entry
 835         options.entries_proc_limit = 1
 836     options.svn_rev_start = None
 837     options.svn_rev_end   = None
 838     if options.svn_rev:
 839         rev = options.svn_rev.split(":")
 840         options.svn_rev_start = rev[0] if len(rev)>0 else None
 841         options.svn_rev_end   = rev[1] if len(rev)>1 else None
 842     ui.update_config(options)
 843     return real_main(options, args)
 844
 845
 846 if __name__ == "__main__":
 847     sys.exit(main() or 0)