svn2svn/run/svn2svn.py

   1 """
   2 Replicate (replay) changesets from one SVN repository to another.
   3 """
   4
   5 from .. import base_version, full_version
   6 from .. import ui
   7 from .. import svnclient
   8 from ..shell import run_svn
   9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
  10 from parse import HelpFormatter
  11
  12 import sys
  13 import os
  14 import time
  15 import traceback
  16 import shutil
  17 import operator
  18 import optparse
  19 import re
  20 from datetime import datetime
  21
  22 _valid_svn_actions = "MARD"   # The list of known SVN action abbr's, from "svn log"
  23
  24 # Module-level variables/parameters
  25 source_url = ""          # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
  26 source_repos_url = ""    # URL to root of source SVN repo,        e.g. 'http://server/svn/source'
  27 source_base = ""         # Relative path of source_url in source SVN repo, e.g. '/trunk'
  28 source_repos_uuid = ""   # UUID of source SVN repo
  29 target_url =""           # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
  30 rev_map = {}             # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
  31 options = None           # optparser options
  32
  33 def commit_from_svn_log_entry(log_entry, commit_paths=None, target_revprops=None):
  34     """
  35     Given an SVN log entry and an optional list of changed paths, do an svn commit.
  36     """
  37     # TODO: Run optional external shell hook here, for doing pre-commit filtering
  38     # Display the _wc_target "svn status" info if running in -vv (or higher) mode
  39     if ui.get_level() >= ui.EXTRA:
  40         ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
  41         ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
  42     # This will use the local timezone for displaying commit times
  43     timestamp = int(log_entry['date'])
  44     svn_date = str(datetime.fromtimestamp(timestamp))
  45     # Uncomment this one one if you prefer UTC commit times
  46     #svn_date = "%d 0" % timestamp
  47     args = ["commit", "--force-log"]
  48     message = log_entry['message']
  49     if options.log_date:
  50         message += "\nDate: " + svn_date
  51     if options.log_author:
  52         message += "\nAuthor: " + log_entry['author']
  53     if options.keep_author:
  54         args += ["--username", log_entry['author']]
  55     args += ["-m", message]
  56     revprops = {}
  57     if log_entry['revprops']:
  58         # Carry forward any revprop's from the source revision
  59         for v in log_entry['revprops']:
  60             revprops[v['name']] = v['value']
  61     if target_revprops:
  62         # Add any extra revprop's we want to set for the target repo commits
  63         for v in target_revprops:
  64             revprops[v['name']] = v['value']
  65     if revprops:
  66         for key in revprops:
  67             args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
  68     if commit_paths:
  69         if len(commit_paths)<100:
  70             # If we don't have an excessive amount of individual changed paths, pass
  71             # those to the "svn commit" command. Else, pass nothing so we commit at
  72             # the root of the working-copy.
  73             args += list(commit_paths)
  74     rev = None
  75     if not options.dry_run:
  76         # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
  77         output = run_svn(args)
  78         if output:
  79             output_lines = output.strip("\n").split("\n")
  80             rev = ""
  81             for line in output_lines:
  82                 if line[0:19] == 'Committed revision ':
  83                     rev = line[19:].rstrip('.')
  84                     break
  85             if rev:
  86                 ui.status("Committed revision %s.", rev)
  87     return rev
  88
  89 def full_svn_revert():
  90     """
  91     Do an "svn revert" and proactively remove any extra files in the working copy.
  92     """
  93     run_svn(["revert", "--recursive", "."])
  94     output = run_svn(["status"])
  95     if output:
  96         output_lines = output.strip("\n").split("\n")
  97         for line in output_lines:
  98             if line[0] == "?":
  99                 path = line[4:].strip(" ")
 100                 if os.path.isfile(path):
 101                     os.remove(path)
 102                 if os.path.isdir(path):
 103                     shutil.rmtree(path)
 104
 105 def gen_tracking_revprops(source_rev):
 106     """
 107     Build an array of svn2svn-specific source-tracking revprops.
 108     """
 109     revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
 110                 {'name':'svn2svn:source_url',  'value':source_url},
 111                 {'name':'svn2svn:source_rev',  'value':source_rev}]
 112     return revprops
 113
 114 def sync_svn_props(source_url, source_rev, path_offset):
 115     """
 116     Carry-forward any unversioned properties from the source repo to the
 117     target WC.
 118     """
 119     source_props = svnclient.get_all_props(source_url+"/"+path_offset,  source_rev)
 120     target_props = svnclient.get_all_props(path_offset)
 121     if 'svn:mergeinfo' in source_props:
 122         # Never carry-forward "svn:mergeinfo"
 123         del source_props['svn:mergeinfo']
 124     for prop in target_props:
 125         if prop not in source_props:
 126             # Remove any properties which exist in target but not source
 127             run_svn(["propdel", prop, path_offset])
 128     for prop in source_props:
 129         if prop not in target_props or \
 130            source_props[prop] != target_props[prop]:
 131             # Set/update any properties which exist in source but not target or
 132             # whose value differs between source vs. target.
 133             run_svn(["propset", prop, source_props[prop], path_offset])
 134
 135 def in_svn(p, require_in_repo=False, prefix=""):
 136     """
 137     Check if a given file/folder is being tracked by Subversion.
 138     Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
 139     With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
 140     Use "svn status" to check the status of the file/folder.
 141     """
 142     entries = svnclient.get_svn_status(p, no_recursive=True)
 143     if not entries:
 144         return False
 145     d = entries[0]
 146     if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
 147         # If caller requires this path to be in the SVN repo, prevent returning True
 148         # for paths that are only locally-added.
 149         ret = False
 150     else:
 151         # Don't consider files tracked as deleted in the WC as under source-control.
 152         # Consider files which are locally added/copied as under source-control.
 153         ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
 154     ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
 155     return ret
 156
 157 def is_child_path(path, p_path):
 158     return True if (path == p_path) or (path.startswith(p_path+"/")) else False
 159
 160 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
 161     """
 162     Given a source path, walk the SVN history backwards to inspect the ancestory of
 163     that path, seeing if it traces back to base_path.  Build an array of copyfrom_path
 164     and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
 165     which base_path is a substring match of (e.g. we crawled back to the initial branch-
 166     copy from trunk), then return the collection of ancestor paths.  Otherwise,
 167     copyfrom_path has no ancestory compared to base_path.
 168
 169     This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
 170     file/folder was renamed in a branch and then that branch was merged back to trunk.
 171
 172     'svn_repos_url' is the full URL to the root of the SVN repository,
 173       e.g. 'file:///path/to/repo'
 174     'base_path' is the path in the SVN repo to the target path we're trying to
 175       trace ancestry back to, e.g. 'trunk'.
 176     'source_path' is the path in the SVN repo to the source path to start checking
 177       ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
 178       (full_path = svn_repos_url+base_path+"/"+path_offset)
 179     'source_rev' is the revision to start walking the history of source_path backwards from.
 180     """
 181     ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s  base_path: %s",
 182         svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
 183     done = False
 184     working_path = base_path+"/"+source_path
 185     working_rev  = source_rev
 186     first_iter_done = False
 187     ancestors_temp = []
 188     while not done:
 189         # Get the first "svn log" entry for this path (relative to @rev)
 190         ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
 191         log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path, 1, working_rev, True)
 192         if not log_entry:
 193             ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
 194             done = True
 195             break
 196         # If we found a copy-from case which matches our base_path, we're done.
 197         # ...but only if we've at least tried to search for the first copy-from path.
 198         if first_iter_done and is_child_path(working_path, base_path):
 199             ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(working_path, base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
 200             done = True
 201             break
 202         first_iter_done = True
 203         # Search for any actions on our target path (or parent paths).
 204         changed_paths_temp = []
 205         for d in log_entry['changed_paths']:
 206             path = d['path']
 207             if path in working_path:
 208                 changed_paths_temp.append({'path': path, 'data': d})
 209         if not changed_paths_temp:
 210             # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
 211             ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
 212             done = True
 213             continue
 214         # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
 215         changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
 216         # Find the action for our working_path in this revision. Use a loop to check in reverse order,
 217         # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
 218         for v in changed_paths:
 219             d = v['data']
 220             path = d['path']
 221             # Check action-type for this file
 222             action = d['action']
 223             if action not in _valid_svn_actions:
 224                 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 225                     % (log_entry['revision'], action))
 226             ui.status(prefix + "> %s %s%s", action, path,
 227                 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 228                 level=ui.DEBUG, color='YELLOW')
 229             if action == 'D':
 230                 # If file/folder was deleted, it has no ancestor
 231                 ancestors_temp = []
 232                 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
 233                 done = True
 234                 break
 235             if action in 'RA':
 236                 # If file/folder was added/replaced but not a copy, it has no ancestor
 237                 if not d['copyfrom_path']:
 238                     ancestors_temp = []
 239                     ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
 240                         "Added" if action == "A" else "Replaced",
 241                         level=ui.DEBUG, color='YELLOW')
 242                     done = True
 243                     break
 244                 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
 245                 # and keep checking for ancestors
 246                 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
 247                     action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
 248                     level=ui.DEBUG, color='YELLOW')
 249                 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
 250                                        'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
 251                 working_path = working_path.replace(d['path'], d['copyfrom_path'])
 252                 working_rev =  d['copyfrom_revision']
 253                 # Follow the copy and keep on searching
 254                 break
 255     ancestors = []
 256     if ancestors_temp:
 257         ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
 258         working_path = base_path+"/"+source_path
 259         for idx in range(len(ancestors_temp)):
 260             d = ancestors_temp[idx]
 261             working_path = working_path.replace(d['path'], d['copyfrom_path'])
 262             working_rev =  d['copyfrom_rev']
 263             ancestors.append({'path': working_path, 'revision': working_rev})
 264         if ui.get_level() >= ui.DEBUG:
 265             max_len = 0
 266             for idx in range(len(ancestors)):
 267                 d = ancestors[idx]
 268                 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
 269             ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
 270             for idx in range(len(ancestors)-1):
 271                 d = ancestors[idx]
 272                 d_next = ancestors[idx+1]
 273                 ui.status(prefix + " [%s] %s <-- %s", idx,
 274                     str(d['path']+"@"+str(d['revision'])).ljust(max_len),
 275                     str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
 276                     level=ui.DEBUG, color='YELLOW')
 277     else:
 278         ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
 279             svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
 280     return ancestors
 281
 282 def get_rev_map(source_rev, prefix):
 283     """
 284     Find the equivalent rev # in the target repo for the given rev # from the source repo.
 285     """
 286     ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
 287     # Find the highest entry less-than-or-equal-to source_rev
 288     for rev in range(int(source_rev), 0, -1):
 289         ui.status(prefix + ">> get_rev_map: rev=%s  in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
 290         if rev in rev_map:
 291             return int(rev_map[rev])
 292     # Else, we fell off the bottom of the rev_map. Ruh-roh...
 293     return None
 294
 295 def set_rev_map(source_rev, target_rev):
 296     ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
 297     global rev_map
 298     rev_map[int(source_rev)]=int(target_rev)
 299
 300 def build_rev_map(target_url, target_end_rev, source_info):
 301     """
 302     Check for any already-replayed history from source_url (source_info) and
 303     build the mapping-table of source_rev -> target_rev.
 304     """
 305     global rev_map
 306     rev_map = {}
 307     ui.status("Rebuilding target_rev -> source_rev rev_map...", level=ui.VERBOSE)
 308     proc_count = 0
 309     it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
 310     for log_entry in it_log_entries:
 311         if log_entry['revprops']:
 312             revprops = {}
 313             for v in log_entry['revprops']:
 314                 if v['name'].startswith('svn2svn:'):
 315                     revprops[v['name']] = v['value']
 316             if revprops and \
 317                revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
 318                revprops['svn2svn:source_url'] == source_info['url']:
 319                 source_rev = revprops['svn2svn:source_rev']
 320                 target_rev = log_entry['revision']
 321                 set_rev_map(source_rev, target_rev)
 322
 323 def get_svn_dirlist(svn_path, rev_number = ""):
 324     """
 325     Get a list of all the child contents (recusive) of the given folder path.
 326     """
 327     args = ["list"]
 328     path = svn_path
 329     if rev_number:
 330         args += ["-r", rev_number]
 331         path += "@"+str(rev_number)
 332     args += [path]
 333     paths = run_svn(args, no_fail=True)
 334     paths = paths.strip("\n").split("\n") if len(paths)>1 else []
 335     return paths
 336
 337 def path_in_list(paths, path):
 338     for p in paths:
 339         if is_child_path(path, p):
 340             return True
 341     return False
 342
 343 def add_path(paths, path):
 344     if not path_in_list(paths, path):
 345         paths.append(path)
 346
 347 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
 348                export_paths={}, is_dir = False, skip_paths=[], prefix = ""):
 349     """
 350     Given the add'd source path, replay the "svn add/copy" commands to correctly
 351     track renames across copy-from's.
 352
 353     For example, consider a sequence of events like this:
 354     1. svn copy /trunk /branches/fix1
 355     2. (Make some changes on /branches/fix1)
 356     3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2  " Rename folder
 357     4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt  " Rename file inside renamed folder
 358     5. svn co /trunk && svn merge /branches/fix1
 359     After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
 360     and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
 361     to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
 362     we'd lose the logical history that Proj2/file2.txt is really a descendant
 363     of Proj1/file1.txt.
 364
 365     'path_offset' is the offset from source_base to the file to check ancestry for,
 366       e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
 367     'source_rev' is the revision ("svn log") that we're processing from the source repo.
 368     'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
 369       directory, when being called recursively by do_svn_add_dir().
 370     'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
 371     'is_dir' is whether path_offset is a directory (rather than a file).
 372     """
 373     ui.status(prefix + ">> do_svn_add: %s  %s", source_base+"/"+path_offset+"@"+str(source_rev),
 374         "  (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
 375         level=ui.DEBUG, color='GREEN')
 376     # Check if the given path has ancestors which chain back to the current source_base
 377     found_ancestor = False
 378     ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+"  ")
 379     # ancestors[n] is the original (pre-branch-copy) trunk path.
 380     # ancestors[n-1] is the first commit on the new branch.
 381     copyfrom_path = ancestors[len(ancestors)-1]['path']     if ancestors else ""
 382     copyfrom_rev  = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
 383     if ancestors:
 384         # The copy-from path has ancestory back to source_url.
 385         ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
 386             level=ui.DEBUG, color='GREEN', bold=True)
 387         found_ancestor = True
 388         # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
 389         # return None in the case where copyfrom_rev is *before* our source_start_rev.
 390         tgt_rev = get_rev_map(copyfrom_rev, prefix+"  ")
 391         ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
 392     else:
 393         ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
 394         found_ancestor = False
 395     if found_ancestor and tgt_rev:
 396         # Check if this path_offset in the target WC already has this ancestry, in which
 397         # case there's no need to run the "svn copy" (again).
 398         path_in_svn = in_svn(path_offset, prefix=prefix+"  ")
 399         log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+"  ") else []
 400         if (not log_entry or (log_entry['revision'] != tgt_rev)):
 401             copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
 402             ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 403             ui.status(prefix + "   copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 404             ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
 405             if path_in_svn and \
 406                ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
 407                 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
 408                 # When being called recursively, if this child entry has the same ancestor as the
 409                 # the parent, then no need to try to run another "svn copy".
 410                 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
 411                     parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
 412                 pass
 413             else:
 414                 # Copy this path from the equivalent path+rev in the target repo, to create the
 415                 # equivalent history.
 416                 if parent_copyfrom_path:
 417                     # If we have a parent copy-from path, we mis-match that so display a status
 418                     # message describing the action we're mimic'ing. If path_in_svn, then this
 419                     # is logically a "replace" rather than an "add".
 420                     ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
 421                 if path_in_svn:
 422                     # If local file is already under version-control, then this is a replace.
 423                     ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
 424                     run_svn(["remove", "--force", path_offset])
 425                 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
 426                 if is_dir:
 427                     # Export the final verison of all files in this folder.
 428                     add_path(export_paths, path_offset)
 429                 else:
 430                     # Export the final verison of this file.
 431                     run_svn(["export", "--force", "-r", source_rev,
 432                              source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 433                 if options.keep_prop:
 434                     sync_svn_props(source_url, source_rev, path_offset)
 435         else:
 436             ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
 437     else:
 438         # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
 439         # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
 440         # Create (parent) directory if needed.
 441         # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
 442         #       split-out to a shared tag?
 443         p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
 444         if not os.path.exists(p_path):
 445             run_svn(["mkdir", p_path])
 446         if not in_svn(path_offset, prefix=prefix+"  "):
 447             if is_dir:
 448                 # Export the final verison of all files in this folder.
 449                 add_path(export_paths, path_offset)
 450             else:
 451                 # Export the final verison of this file. We *need* to do this before running
 452                 # the "svn add", even if we end-up re-exporting this file again via export_paths.
 453                 run_svn(["export", "--force", "-r", source_rev,
 454                          source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 455             # If not already under version-control, then "svn add" this file/folder.
 456             run_svn(["add", "--parents", path_offset])
 457         if options.keep_prop:
 458             sync_svn_props(source_url, source_rev, path_offset)
 459     if is_dir:
 460         # For any folders that we process, process any child contents, so that we correctly
 461         # replay copies/replaces/etc.
 462         do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, skip_paths, prefix+"  ")
 463
 464 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
 465                    export_paths, skip_paths, prefix=""):
 466     # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
 467     # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
 468     #       associated remote repo folder. (Is this a problem?)
 469     paths_local =  get_svn_dirlist(path_offset)
 470     paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
 471     ui.status(prefix + ">> do_svn_add_dir: paths_local:  %s", str(paths_local),  level=ui.DEBUG, color='GREEN')
 472     ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
 473     # Update files/folders which exist in remote but not local
 474     for path in paths_remote:
 475         path_is_dir = True if path[-1] == "/" else False
 476         working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
 477         if not working_path in skip_paths:
 478             do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
 479                        export_paths, path_is_dir, skip_paths, prefix+"  ")
 480     # Remove files/folders which exist in local but not remote
 481     for path in paths_local:
 482         if not path in paths_remote:
 483             ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
 484             run_svn(["remove", "--force", path_offset+"/"+path])
 485             # TODO: Does this handle deleted folders too? Wouldn't want to have a case
 486             #       where we only delete all files from folder but leave orphaned folder around.
 487
 488 def process_svn_log_entry(log_entry, commit_paths, prefix = ""):
 489     """
 490     Process SVN changes from the given log entry. Build an array (commit_paths)
 491     of the paths in the working-copy that were changed, i.e. the paths which
 492     we'll pass to "svn commit".
 493     """
 494     export_paths = []
 495     source_rev = log_entry['revision']
 496     ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
 497     for d in log_entry['changed_paths']:
 498         # Get the full path for this changed_path
 499         # e.g. '/branches/bug123/projectA/file1.txt'
 500         path = d['path']
 501         if not is_child_path(path, source_base):
 502             # Ignore changed files that are not part of this subdir
 503             ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s  (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
 504             continue
 505         if d['kind'] == "":
 506             # The "kind" value was introduced in SVN 1.6, and "svn log --xml" won't return a "kind"
 507             # value for commits made on a pre-1.6 repo, even if the server is now running 1.6.
 508             # We need to use other methods to fetch the node-kind for these cases.
 509             d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
 510         assert (d['kind'] == 'file') or (d['kind'] == 'dir')
 511         path_is_dir =  True if d['kind'] == 'dir'  else False
 512         path_is_file = True if d['kind'] == 'file' else False
 513         # Calculate the offset (based on source_base) for this changed_path
 514         # e.g. 'projectA/file1.txt'
 515         # (path = source_base + "/" + path_offset)
 516         path_offset = path[len(source_base):].strip("/")
 517         # Get the action for this path
 518         action = d['action']
 519         if action not in _valid_svn_actions:
 520             raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 521                 % (source_rev, action))
 522         ui.status(" %s %s%s", action, d['path'],
 523             (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 524             level=ui.VERBOSE)
 525
 526         # Try to be efficient and keep track of an explicit list of paths in the
 527         # working copy that changed. If we commit from the root of the working copy,
 528         # then SVN needs to crawl the entire working copy looking for pending changes.
 529         commit_paths.append(path_offset)
 530
 531         # Special-handling for replace's
 532         if action == 'R':
 533             # If file was "replaced" (deleted then re-added, all in same revision),
 534             # then we need to run the "svn rm" first, then change action='A'. This
 535             # lets the normal code below handle re-"svn add"'ing the files. This
 536             # should replicate the "replace".
 537             if in_svn(path_offset):
 538                 # Target path might not be under version-control yet, e.g. parent "add"
 539                 # was a copy-from a branch which had no ancestry back to trunk, and each
 540                 # child folder under that parent folder is a "replace" action on the final
 541                 # merge to trunk. Since the child folders will be in skip_paths, do_svn_add
 542                 # wouldn't have created them while processing the parent "add" path.
 543                 run_svn(["remove", "--force", path_offset])
 544             action = 'A'
 545
 546         # Handle all the various action-types
 547         # (Handle "add" first, for "svn copy/move" support)
 548         if action == 'A':
 549             # Determine where to export from.
 550             svn_copy = False
 551             # Handle cases where this "add" was a copy from another URL in the source repo
 552             if d['copyfrom_revision']:
 553                 copyfrom_path = d['copyfrom_path']
 554                 copyfrom_rev =  d['copyfrom_revision']
 555                 skip_paths = []
 556                 for tmp_d in log_entry['changed_paths']:
 557                     tmp_path = tmp_d['path']
 558                     if is_child_path(tmp_path, path):
 559                         # Build list of child entries which are also in the changed_paths list,
 560                         # so that do_svn_add() can skip processing these entries when recursing
 561                         # since we'll end-up processing them later.
 562                         tmp_path_offset = tmp_path[len(source_base):].strip("/")
 563                         skip_paths.append(tmp_path_offset)
 564                 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, skip_paths, prefix+"  ")
 565             # Else just "svn export" the files from the source repo and "svn add" them.
 566             else:
 567                 # Create (parent) directory if needed
 568                 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
 569                 if not os.path.exists(p_path):
 570                     run_svn(["mkdir", p_path])
 571                 # Export the entire added tree.
 572                 if path_is_dir:
 573                     # For directories, defer the (recurisve) "svn export". Might have a
 574                     # situation in a branch merge where the entry in the svn-log is a
 575                     # non-copy-from'd "add" but there are child contents (that we haven't
 576                     # gotten to yet in log_entry) that are copy-from's.  When we try do
 577                     # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
 578                     # having pre-existing (svn-add'd) contents creates some trouble.
 579                     # Instead, just create the stub folders ("svn mkdir" above) and defer
 580                     # exporting the final file-state until the end.
 581                     add_path(export_paths, path_offset)
 582                 else:
 583                     # Export the final verison of this file. We *need* to do this before running
 584                     # the "svn add", even if we end-up re-exporting this file again via export_paths.
 585                     run_svn(["export", "--force", "-r", source_rev,
 586                              source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 587                 if not in_svn(path_offset, prefix=prefix+"  "):
 588                     # Need to use in_svn here to handle cases where client committed the parent
 589                     # folder and each indiv sub-folder.
 590                     run_svn(["add", "--parents", path_offset])
 591                 if options.keep_prop:
 592                     sync_svn_props(source_url, source_rev, path_offset)
 593
 594         elif action == 'D':
 595             run_svn(["remove", "--force", path_offset])
 596
 597         elif action == 'M':
 598             if path_is_file:
 599                 run_svn(["export", "--force", "-N" , "-r", source_rev,
 600                          source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 601             if options.keep_prop:
 602                 sync_svn_props(source_url, source_rev, path_offset)
 603
 604         else:
 605             raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
 606                 % action)
 607
 608     # Export the final version of all add'd paths from source_url
 609     if export_paths:
 610         for path_offset in export_paths:
 611             run_svn(["export", "--force", "-r", source_rev,
 612                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 613
 614 def disp_svn_log_summary(log_entry):
 615     ui.status("------------------------------------------------------------------------")
 616     ui.status("r%s | %s | %s",
 617         log_entry['revision'],
 618         log_entry['author'],
 619         str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
 620     ui.status(log_entry['message'])
 621
 622 def real_main(args, parser):
 623     global source_url, target_url, rev_map
 624     source_url = args.pop(0).rstrip("/")    # e.g. 'http://server/svn/source/trunk'
 625     target_url = args.pop(0).rstrip("/")    # e.g. 'file:///svn/target/trunk'
 626     ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
 627
 628     # Make sure that both the source and target URL's are valid
 629     source_info = svnclient.get_svn_info(source_url)
 630     assert is_child_path(source_url, source_info['repos_url'])
 631     target_info = svnclient.get_svn_info(target_url)
 632     assert is_child_path(target_url, target_info['repos_url'])
 633
 634     # Init global vars
 635     global source_repos_url,source_base,source_repos_uuid
 636     source_repos_url = source_info['repos_url']       # e.g. 'http://server/svn/source'
 637     source_base = source_url[len(source_repos_url):]  # e.g. '/trunk'
 638     source_repos_uuid = source_info['repos_uuid']
 639
 640     # Init start and end revision
 641     try:
 642         source_start_rev = svnclient.get_svn_rev(source_repos_url, options.rev_start if options.rev_start else 1)
 643     except ExternalCommandFailed:
 644         parser.error("invalid start source revision value: %s" % (options.rev_start))
 645     try:
 646         source_end_rev   = svnclient.get_svn_rev(source_repos_url, options.rev_end   if options.rev_end   else "HEAD")
 647     except ExternalCommandFailed:
 648         parser.error("invalid end source revision value: %s" % (options.rev_end))
 649     ui.status("Using source revision range %s:%s", source_start_rev, source_end_rev, level=ui.VERBOSE)
 650
 651     # TODO: If options.keep_date, should we try doing a "svn propset" on an *existing* revision
 652     #       as a sanity check, so we check if the pre-revprop-change hook script is correctly setup
 653     #       before doing first replay-commit?
 654
 655     target_end_rev = target_info['revision']   # Last revision # in the target repo
 656     wc_target = os.path.abspath('_wc_target')
 657     num_entries_proc = 0
 658     commit_count = 0
 659     source_rev = None
 660     target_rev = None
 661
 662     # Check out a working copy of target_url if needed
 663     wc_exists = os.path.exists(wc_target)
 664     if wc_exists and not options.cont_from_break:
 665         shutil.rmtree(wc_target)
 666         wc_exists = False
 667     if not wc_exists:
 668         ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
 669         svnclient.svn_checkout(target_url, wc_target)
 670     os.chdir(wc_target)
 671
 672     if not options.cont_from_break:
 673         # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
 674         # Get the first log entry at/after source_start_rev, which is where
 675         # we'll do the initial import from.
 676         it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False)
 677         for source_start_log in it_log_start:
 678             break
 679         if not source_start_log:
 680             raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
 681                 (source_start_rev, source_end_rev, source_url))
 682
 683         # This is the revision we will start from for source_url
 684         source_start_rev = source_rev = int(source_start_log['revision'])
 685         ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
 686         ui.status("")
 687
 688         # For the initial commit to the target URL, export all the contents from
 689         # the source URL at the start-revision.
 690         disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
 691         ui.status("(Initial import)", level=ui.VERBOSE)
 692         # Export and add file-contents from source_url@source_start_rev
 693         top_paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
 694         top_paths = top_paths.strip("\n").split("\n")
 695         for path in top_paths:
 696             # For each top-level file/folder...
 697             if not path:
 698                 continue
 699             # Directories have a trailing slash in the "svn list" output
 700             path_is_dir = True if path[-1] == "/" else False
 701             path_offset = path.rstrip('/') if path_is_dir else path
 702             if in_svn(path_offset, prefix="  "):
 703                 raise InternalError("Cannot replay history on top of pre-existing structure: %s" % source_url+"/"+path_offset)
 704             if path_is_dir and not os.path.exists(path_offset):
 705                 os.makedirs(path_offset)
 706             run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 707             run_svn(["add", path_offset])
 708         # Update any properties on the newly added content
 709         paths = run_svn(["list", "--recursive", "-r", source_rev, source_url+"@"+str(source_rev)])
 710         paths = paths.strip("\n").split("\n")
 711         if options.keep_prop:
 712             sync_svn_props(source_url, source_rev, "")
 713         for path in paths:
 714             if not path:
 715                 continue
 716             # Directories have a trailing slash in the "svn list" output
 717             path_is_dir = True if path[-1] == "/" else False
 718             path_offset = path.rstrip('/') if path_is_dir else path
 719             ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path_offset, level=ui.VERBOSE)
 720             if options.keep_prop:
 721                 sync_svn_props(source_url, source_rev, path_offset)
 722         # Commit the initial import
 723         num_entries_proc += 1
 724         target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 725         target_rev = commit_from_svn_log_entry(source_start_log, target_revprops=target_revprops)
 726         if target_rev:
 727             # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 728             set_rev_map(source_rev, target_rev)
 729             # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 730             run_svn(["update"])
 731             commit_count += 1
 732     else:
 733         # Re-build the rev_map based on any already-replayed history in target_url
 734         build_rev_map(target_url, target_end_rev, source_info)
 735         if not rev_map:
 736             parser.error("called with continue-mode, but no already-replayed source history found in target_url")
 737         source_start_rev = int(max(rev_map, key=rev_map.get))
 738         assert source_start_rev
 739         ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
 740         ui.status("")
 741
 742     svn_vers_t = svnclient.get_svn_client_version()
 743     svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
 744
 745     # Load SVN log starting from source_start_rev + 1
 746     it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else []
 747     source_rev = None
 748
 749     # TODO: Now that commit_from_svn_log_entry() might try to do a "svn propset svn:date",
 750     #       we might want some better KeyboardInterupt handilng here, to ensure that
 751     #       commit_from_svn_log_entry() always runs as an atomic unit.
 752     try:
 753         for log_entry in it_log_entries:
 754             if options.entries_proc_limit:
 755                 if num_entries_proc >= options.entries_proc_limit:
 756                     break
 757             # Replay this revision from source_url into target_url
 758             disp_svn_log_summary(log_entry)
 759             source_rev = log_entry['revision']
 760             # Process all the changed-paths in this log entry
 761             commit_paths = []
 762             process_svn_log_entry(log_entry, commit_paths)
 763             num_entries_proc += 1
 764             # Commit any changes made to _wc_target
 765             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 766             target_rev = commit_from_svn_log_entry(log_entry, commit_paths, target_revprops=target_revprops)
 767             if target_rev:
 768                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 769                 source_rev = log_entry['revision']
 770                 set_rev_map(source_rev, target_rev)
 771                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 772                 run_svn(["update"])
 773                 commit_count += 1
 774                 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
 775                 if svn_vers >= 1.7 and (commit_count % 100 == 0):
 776                     run_svn(["cleanup"])
 777         if not source_rev:
 778             # If there were no new source_url revisions to process, init source_rev
 779             # for the "finally" message below.
 780             source_rev = source_end_rev
 781
 782     except KeyboardInterrupt:
 783         print "\nStopped by user."
 784         print "\nCleaning-up..."
 785         run_svn(["cleanup"])
 786         full_svn_revert()
 787     except:
 788         print "\nCommand failed with following error:\n"
 789         traceback.print_exc()
 790         print "\nCleaning-up..."
 791         run_svn(["cleanup"])
 792         print run_svn(["status"])
 793         full_svn_revert()
 794     finally:
 795         print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
 796
 797 def main():
 798     # Defined as entry point. Must be callable without arguments.
 799     usage = "svn2svn, version %s\n" % str(full_version) + \
 800             "<http://nynim.org/projects/svn2svn> <https://github.com/tonyduckles/svn2svn>\n\n" + \
 801             "Usage: %prog [OPTIONS] source_url target_url\n"
 802     description = """\
 803 Replicate (replay) history from one SVN repository to another. Maintain
 804 logical ancestry wherever possible, so that 'svn log' on the replayed repo
 805 will correctly follow file/folder renames.
 806
 807 Examples:
 808   Create a copy of only /trunk from source repo, starting at r5000
 809   $ svnadmin create /svn/target
 810   $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
 811   $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
 812     1. The target_url will be checked-out to ./_wc_target
 813     2. The first commit to http://server/source/trunk at/after r5000 will be
 814        exported & added into _wc_target
 815     3. All revisions affecting http://server/source/trunk (starting at r5000)
 816        will be replayed to _wc_target. Any add/copy/move/replaces that are
 817        copy-from'd some path outside of /trunk (e.g. files renamed on a
 818        /branch and branch was merged into /trunk) will correctly maintain
 819        logical ancestry where possible.
 820
 821   Use continue-mode (-c) to pick-up where the last run left-off
 822   $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
 823     1. The target_url will be checked-out to ./_wc_target, if not already
 824        checked-out
 825     2. All new revisions affecting http://server/source/trunk starting from
 826        the last replayed revision to file:///svn/target/trunk (based on the
 827        svn2svn:* revprops) will be replayed to _wc_target, maintaining all
 828        logical ancestry where possible."""
 829     parser = optparse.OptionParser(usage, description=description,
 830                 formatter=HelpFormatter(), version="%prog "+str(full_version))
 831     parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
 832                       help="enable additional output (use -vv or -vvv for more)")
 833     parser.add_option("-a", "--archive", action="store_true", dest="archive", default=False,
 834                       help="archive/mirror mode; same as -UDP (see REQUIRE's below)\n"
 835                            "maintain same commit author, same commit time, and file/dir properties")
 836     parser.add_option("-U", "--keep-author", action="store_true", dest="keep_author", default=False,
 837                       help="maintain same commit authors (svn:author) as source\n"
 838                            "(REQUIRES target_url be non-auth'd, e.g. file://-based, since this uses --username to set author)")
 839     parser.add_option("-D", "--keep-date", action="store_true", dest="keep_date", default=False,
 840                       help="maintain same commit time (svn:date) as source\n"
 841                            "(REQUIRES 'pre-revprop-change' hook script to allow 'svn:date' changes)")
 842     parser.add_option("-P", "--keep-prop", action="store_true", dest="keep_prop", default=False,
 843                       help="maintain same file/dir SVN properties as source")
 844     parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
 845                       help="continue from last source commit to target (based on svn2svn:* revprops)")
 846     parser.add_option("-r", "--revision", type="string", dest="revision", metavar="ARG",
 847                       help="revision range to replay from source_url\n"
 848                            "A revision argument can be one of:\n"
 849                            "   START        start rev # (end will be 'HEAD')\n"
 850                            "   START:END    start and ending rev #'s\n"
 851                            "Any revision # formats which SVN understands are "
 852                            "supported, e.g. 'HEAD', '{2010-01-31}', etc.")
 853     parser.add_option("-u", "--log-author", action="store_true", dest="log_author", default=False,
 854                       help="append source commit author to replayed commit mesages")
 855     parser.add_option("-d", "--log-date", action="store_true", dest="log_date", default=False,
 856                       help="append source commit time to replayed commit messages")
 857     parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
 858                       help="maximum number of source revisions to process")
 859     parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 860                       help="process next source revision but don't commit changes to "
 861                            "target working-copy (forces --limit=1)")
 862     parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
 863                       help="enable debugging output (same as -vvv)")
 864     global options
 865     options, args = parser.parse_args()
 866     if len(args) != 2:
 867         parser.error("incorrect number of arguments")
 868     if options.verbosity < 10:
 869         # Expand multiple "-v" arguments to a real ui._level value
 870         options.verbosity *= 10
 871     if options.dry_run:
 872         # When in dry-run mode, only try to process the next log_entry
 873         options.entries_proc_limit = 1
 874     options.rev_start = None
 875     options.rev_end   = None
 876     if options.revision:
 877         # Reg-ex for matching a revision arg (http://svnbook.red-bean.com/en/1.5/svn.tour.revs.specifiers.html#svn.tour.revs.dates)
 878         rev_patt = '[0-9A-Z]+|\{[0-9A-Za-z/\\ :-]+\}'
 879         rev = None
 880         match = re.match('^('+rev_patt+'):('+rev_patt+')$', options.revision)  # First try start:end match
 881         if match is None: match = re.match('^('+rev_patt+')$', options.revision)   # Next, try start match
 882         if match is None:
 883             parser.error("unexpected --revision argument format; see 'svn help log' for valid revision formats")
 884         rev = match.groups()
 885         options.rev_start = rev[0] if len(rev)>0 else None
 886         options.rev_end   = rev[1] if len(rev)>1 else None
 887     if options.archive:
 888         options.keep_author = True
 889         options.keep_date   = True
 890         options.keep_prop   = True
 891     ui.update_config(options)
 892     return real_main(args, parser)
 893
 894
 895 if __name__ == "__main__":
 896     sys.exit(main() or 0)