svn2svn/run/svn2svn.py

   1 """
   2 Replicate (replay) changesets from one SVN repository to another.
   3 """
   4
   5 from .. import base_version, full_version
   6 from .. import ui
   7 from .. import svnclient
   8 from ..shell import run_svn
   9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
  10 from parse import HelpFormatter
  11
  12 import sys
  13 import os
  14 import time
  15 import traceback
  16 import shutil
  17 import operator
  18 import optparse
  19 from datetime import datetime
  20
  21 _valid_svn_actions = "MARD"   # The list of known SVN action abbr's, from "svn log"
  22
  23 # Module-level variables/parameters
  24 source_url = ""          # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
  25 source_repos_url = ""    # URL to root of source SVN repo,        e.g. 'http://server/svn/source'
  26 source_base = ""         # Relative path of source_url in source SVN repo, e.g. '/trunk'
  27 source_repos_uuid = ""   # UUID of source SVN repo
  28 target_url =""           # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
  29 rev_map = {}             # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
  30
  31 def commit_from_svn_log_entry(log_entry, options, commit_paths=None, target_revprops=None):
  32     """
  33     Given an SVN log entry and an optional list of changed paths, do an svn commit.
  34     """
  35     # TODO: Run optional external shell hook here, for doing pre-commit filtering
  36     # Display the _wc_target "svn status" info if running in -vv (or higher) mode
  37     if ui.get_level() >= ui.EXTRA:
  38         ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
  39         ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
  40     # This will use the local timezone for displaying commit times
  41     timestamp = int(log_entry['date'])
  42     svn_date = str(datetime.fromtimestamp(timestamp))
  43     # Uncomment this one one if you prefer UTC commit times
  44     #svn_date = "%d 0" % timestamp
  45     args = ["commit", "--force-log"]
  46     if options.keep_author:
  47         args += ["-m", log_entry['message'] + "\nDate: " + svn_date, "--username", log_entry['author']]
  48     else:
  49         args += ["-m", log_entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + log_entry['author']]
  50     revprops = {}
  51     if log_entry['revprops']:
  52         # Carry forward any revprop's from the source revision
  53         for v in log_entry['revprops']:
  54             revprops[v['name']] = v['value']
  55     if target_revprops:
  56         # Add any extra revprop's we want to set for the target repo commits
  57         for v in target_revprops:
  58             revprops[v['name']] = v['value']
  59     if revprops:
  60         for key in revprops:
  61             args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
  62     if commit_paths:
  63         if len(commit_paths)<100:
  64             # If we don't have an excessive amount of individual changed paths, pass
  65             # those to the "svn commit" command. Else, pass nothing so we commit at
  66             # the root of the working-copy.
  67             args += list(commit_paths)
  68     rev = None
  69     if not options.dry_run:
  70         # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
  71         output = run_svn(args)
  72         if output:
  73             output_lines = output.strip("\n").split("\n")
  74             rev = ""
  75             for line in output_lines:
  76                 if line[0:19] == 'Committed revision ':
  77                     rev = line[19:].rstrip('.')
  78                     break
  79             if rev:
  80                 ui.status("Committed revision %s.", rev)
  81     return rev
  82
  83 def full_svn_revert():
  84     """
  85     Do an "svn revert" and proactively remove any extra files in the working copy.
  86     """
  87     run_svn(["revert", "--recursive", "."])
  88     output = run_svn(["status"])
  89     if output:
  90         output_lines = output.strip("\n").split("\n")
  91         for line in output_lines:
  92             if line[0] == "?":
  93                 path = line[4:].strip(" ")
  94                 if os.path.isfile(path):
  95                     os.remove(path)
  96                 if os.path.isdir(path):
  97                     shutil.rmtree(path)
  98
  99 def gen_tracking_revprops(source_rev):
 100     """
 101     Build an array of svn2svn-specific source-tracking revprops.
 102     """
 103     revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
 104                 {'name':'svn2svn:source_url',  'value':source_url},
 105                 {'name':'svn2svn:source_rev',  'value':source_rev}]
 106     return revprops
 107
 108 def sync_svn_props(source_url, source_rev, path_offset):
 109     """
 110     Carry-forward any unversioned properties from the source repo to the
 111     target WC.
 112     """
 113     source_props = svnclient.get_all_props(source_url+"/"+path_offset,  source_rev)
 114     target_props = svnclient.get_all_props(path_offset)
 115     if 'svn:mergeinfo' in source_props:
 116         # Never carry-forward "svn:mergeinfo"
 117         del source_props['svn:mergeinfo']
 118     for prop in target_props:
 119         if prop not in source_props:
 120             # Remove any properties which exist in target but not source
 121             run_svn(["propdel", prop, path_offset])
 122     for prop in source_props:
 123         if prop not in target_props or \
 124            source_props[prop] != target_props[prop]:
 125             # Set/update any properties which exist in source but not target or
 126             # whose value differs between source vs. target.
 127             run_svn(["propset", prop, source_props[prop], path_offset])
 128
 129 def in_svn(p, require_in_repo=False, prefix=""):
 130     """
 131     Check if a given file/folder is being tracked by Subversion.
 132     Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
 133     With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
 134     Use "svn status" to check the status of the file/folder.
 135     """
 136     entries = svnclient.get_svn_status(p, no_recursive=True)
 137     if not entries:
 138         return False
 139     d = entries[0]
 140     if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
 141         # If caller requires this path to be in the SVN repo, prevent returning True
 142         # for paths that are only locally-added.
 143         ret = False
 144     else:
 145         # Don't consider files tracked as deleted in the WC as under source-control.
 146         # Consider files which are locally added/copied as under source-control.
 147         ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
 148     ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
 149     return ret
 150
 151 def is_child_path(path, p_path):
 152     return True if (path == p_path) or (path.startswith(p_path+"/")) else False
 153
 154 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
 155     """
 156     Given a source path, walk the SVN history backwards to inspect the ancestory of
 157     that path, seeing if it traces back to base_path.  Build an array of copyfrom_path
 158     and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
 159     which base_path is a substring match of (e.g. we crawled back to the initial branch-
 160     copy from trunk), then return the collection of ancestor paths.  Otherwise,
 161     copyfrom_path has no ancestory compared to base_path.
 162
 163     This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
 164     file/folder was renamed in a branch and then that branch was merged back to trunk.
 165
 166     'svn_repos_url' is the full URL to the root of the SVN repository,
 167       e.g. 'file:///path/to/repo'
 168     'base_path' is the path in the SVN repo to the target path we're trying to
 169       trace ancestry back to, e.g. 'trunk'.
 170     'source_path' is the path in the SVN repo to the source path to start checking
 171       ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
 172       (full_path = svn_repos_url+base_path+"/"+path_offset)
 173     'source_rev' is the revision to start walking the history of source_path backwards from.
 174     """
 175     ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s  base_path: %s",
 176         svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
 177     done = False
 178     working_path = base_path+"/"+source_path
 179     working_rev  = source_rev
 180     first_iter_done = False
 181     ancestors_temp = []
 182     while not done:
 183         # Get the first "svn log" entry for this path (relative to @rev)
 184         ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
 185         log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path, 1, working_rev, True)
 186         if not log_entry:
 187             ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
 188             done = True
 189             break
 190         # If we found a copy-from case which matches our base_path, we're done.
 191         # ...but only if we've at least tried to search for the first copy-from path.
 192         if first_iter_done and is_child_path(working_path, base_path):
 193             ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(working_path, base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
 194             done = True
 195             break
 196         first_iter_done = True
 197         # Search for any actions on our target path (or parent paths).
 198         changed_paths_temp = []
 199         for d in log_entry['changed_paths']:
 200             path = d['path']
 201             if path in working_path:
 202                 changed_paths_temp.append({'path': path, 'data': d})
 203         if not changed_paths_temp:
 204             # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
 205             ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
 206             done = True
 207             continue
 208         # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
 209         changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
 210         # Find the action for our working_path in this revision. Use a loop to check in reverse order,
 211         # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
 212         for v in changed_paths:
 213             d = v['data']
 214             path = d['path']
 215             # Check action-type for this file
 216             action = d['action']
 217             if action not in _valid_svn_actions:
 218                 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 219                     % (log_entry['revision'], action))
 220             ui.status(prefix + "> %s %s%s", action, path,
 221                 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 222                 level=ui.DEBUG, color='YELLOW')
 223             if action == 'D':
 224                 # If file/folder was deleted, it has no ancestor
 225                 ancestors_temp = []
 226                 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
 227                 done = True
 228                 break
 229             if action in 'RA':
 230                 # If file/folder was added/replaced but not a copy, it has no ancestor
 231                 if not d['copyfrom_path']:
 232                     ancestors_temp = []
 233                     ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
 234                         "Added" if action == "A" else "Replaced",
 235                         level=ui.DEBUG, color='YELLOW')
 236                     done = True
 237                     break
 238                 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
 239                 # and keep checking for ancestors
 240                 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
 241                     action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
 242                     level=ui.DEBUG, color='YELLOW')
 243                 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
 244                                        'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
 245                 working_path = working_path.replace(d['path'], d['copyfrom_path'])
 246                 working_rev =  d['copyfrom_revision']
 247                 # Follow the copy and keep on searching
 248                 break
 249     ancestors = []
 250     if ancestors_temp:
 251         ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
 252         working_path = base_path+"/"+source_path
 253         for idx in range(len(ancestors_temp)):
 254             d = ancestors_temp[idx]
 255             working_path = working_path.replace(d['path'], d['copyfrom_path'])
 256             working_rev =  d['copyfrom_rev']
 257             ancestors.append({'path': working_path, 'revision': working_rev})
 258         if ui.get_level() >= ui.DEBUG:
 259             max_len = 0
 260             for idx in range(len(ancestors)):
 261                 d = ancestors[idx]
 262                 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
 263             ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
 264             for idx in range(len(ancestors)-1):
 265                 d = ancestors[idx]
 266                 d_next = ancestors[idx+1]
 267                 ui.status(prefix + " [%s] %s <-- %s", idx,
 268                     str(d['path']+"@"+str(d['revision'])).ljust(max_len),
 269                     str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
 270                     level=ui.DEBUG, color='YELLOW')
 271     else:
 272         ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
 273             svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
 274     return ancestors
 275
 276 def get_rev_map(source_rev, prefix):
 277     """
 278     Find the equivalent rev # in the target repo for the given rev # from the source repo.
 279     """
 280     ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
 281     # Find the highest entry less-than-or-equal-to source_rev
 282     for rev in range(int(source_rev), 0, -1):
 283         ui.status(prefix + ">> get_rev_map: rev=%s  in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
 284         if rev in rev_map:
 285             return int(rev_map[rev])
 286     # Else, we fell off the bottom of the rev_map. Ruh-roh...
 287     return None
 288
 289 def set_rev_map(source_rev, target_rev):
 290     ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
 291     global rev_map
 292     rev_map[int(source_rev)]=int(target_rev)
 293
 294 def build_rev_map(target_url, target_end_rev, source_info):
 295     """
 296     Check for any already-replayed history from source_url (source_info) and
 297     build the mapping-table of source_rev -> target_rev.
 298     """
 299     global rev_map
 300     rev_map = {}
 301     ui.status("Rebuilding rev_map...", level=ui.VERBOSE)
 302     proc_count = 0
 303     it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
 304     for log_entry in it_log_entries:
 305         if log_entry['revprops']:
 306             revprops = {}
 307             for v in log_entry['revprops']:
 308                 if v['name'].startswith('svn2svn:'):
 309                     revprops[v['name']] = v['value']
 310             if revprops and \
 311                revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
 312                revprops['svn2svn:source_url'] == source_info['url']:
 313                 source_rev = revprops['svn2svn:source_rev']
 314                 target_rev = log_entry['revision']
 315                 set_rev_map(source_rev, target_rev)
 316
 317 def get_svn_dirlist(svn_path, svn_rev = ""):
 318     """
 319     Get a list of all the child contents (recusive) of the given folder path.
 320     """
 321     args = ["list"]
 322     path = svn_path
 323     if svn_rev:
 324         args += ["-r", svn_rev]
 325         path += "@"+str(svn_rev)
 326     args += [path]
 327     paths = run_svn(args, no_fail=True)
 328     paths = paths.strip("\n").split("\n") if len(paths)>1 else []
 329     return paths
 330
 331 def path_in_list(paths, path):
 332     for p in paths:
 333         if is_child_path(path, p):
 334             return True
 335     return False
 336
 337 def add_path(paths, path):
 338     if not path_in_list(paths, path):
 339         paths.append(path)
 340
 341 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
 342                export_paths={}, is_dir = False, prefix = ""):
 343     """
 344     Given the add'd source path, replay the "svn add/copy" commands to correctly
 345     track renames across copy-from's.
 346
 347     For example, consider a sequence of events like this:
 348     1. svn copy /trunk /branches/fix1
 349     2. (Make some changes on /branches/fix1)
 350     3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2  " Rename folder
 351     4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt  " Rename file inside renamed folder
 352     5. svn co /trunk && svn merge /branches/fix1
 353     After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
 354     and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
 355     to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
 356     we'd lose the logical history that Proj2/file2.txt is really a descendant
 357     of Proj1/file1.txt.
 358
 359     'path_offset' is the offset from source_base to the file to check ancestry for,
 360       e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
 361     'source_rev' is the revision ("svn log") that we're processing from the source repo.
 362     'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
 363       directory, when being called recursively by do_svn_add_dir().
 364     'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
 365     'is_dir' is whether path_offset is a directory (rather than a file).
 366     """
 367     ui.status(prefix + ">> do_svn_add: %s  %s", source_base+"/"+path_offset+"@"+str(source_rev),
 368         "  (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
 369         level=ui.DEBUG, color='GREEN')
 370     # Check if the given path has ancestors which chain back to the current source_base
 371     found_ancestor = False
 372     ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+"  ")
 373     # ancestors[n] is the original (pre-branch-copy) trunk path.
 374     # ancestors[n-1] is the first commit on the new branch.
 375     copyfrom_path = ancestors[len(ancestors)-1]['path']     if ancestors else ""
 376     copyfrom_rev  = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
 377     if ancestors:
 378         # The copy-from path has ancestory back to source_url.
 379         ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
 380             level=ui.DEBUG, color='GREEN', bold=True)
 381         found_ancestor = True
 382         # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
 383         # return None in the case where copyfrom_rev is *before* our source_start_rev.
 384         tgt_rev = get_rev_map(copyfrom_rev, prefix+"  ")
 385         ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
 386     else:
 387         ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
 388         found_ancestor = False
 389     if found_ancestor and tgt_rev:
 390         # Check if this path_offset in the target WC already has this ancestry, in which
 391         # case there's no need to run the "svn copy" (again).
 392         path_in_svn = in_svn(path_offset, prefix=prefix+"  ")
 393         log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+"  ") else []
 394         if (not log_entry or (log_entry['revision'] != tgt_rev)):
 395             copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
 396             ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 397             ui.status(prefix + "   copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 398             ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
 399             if path_in_svn and \
 400                ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
 401                 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
 402                 # When being called recursively, if this child entry has the same ancestor as the
 403                 # the parent, then no need to try to run another "svn copy".
 404                 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
 405                     parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
 406                 pass
 407             else:
 408                 # Copy this path from the equivalent path+rev in the target repo, to create the
 409                 # equivalent history.
 410                 if parent_copyfrom_path:
 411                     # If we have a parent copy-from path, we mis-match that so display a status
 412                     # message describing the action we're mimic'ing. If path_in_svn, then this
 413                     # is logically a "replace" rather than an "add".
 414                     ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
 415                 if path_in_svn:
 416                     # If local file is already under version-control, then this is a replace.
 417                     ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
 418                     run_svn(["remove", "--force", path_offset])
 419                 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
 420                 if is_dir:
 421                     # Export the final verison of all files in this folder.
 422                     add_path(export_paths, path_offset)
 423                 else:
 424                     # Export the final verison of this file.
 425                     run_svn(["export", "--force", "-r", source_rev,
 426                              source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 427                 # Copy SVN properties from source repo
 428                 sync_svn_props(source_url, source_rev, path_offset)
 429         else:
 430             ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
 431     else:
 432         # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
 433         # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
 434         # Create (parent) directory if needed.
 435         # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
 436         #       split-out to a shared tag?
 437         p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
 438         if not os.path.exists(p_path):
 439             run_svn(["mkdir", p_path])
 440         if not in_svn(path_offset, prefix=prefix+"  "):
 441             if is_dir:
 442                 # Export the final verison of all files in this folder.
 443                 add_path(export_paths, path_offset)
 444             else:
 445                 # Export the final verison of this file. We *need* to do this before running
 446                 # the "svn add", even if we end-up re-exporting this file again via export_paths.
 447                 run_svn(["export", "--force", "-r", source_rev,
 448                          source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 449             # If not already under version-control, then "svn add" this file/folder.
 450             run_svn(["add", "--parents", path_offset])
 451         # Copy SVN properties from source repo
 452         sync_svn_props(source_url, source_rev, path_offset)
 453     if is_dir:
 454         # For any folders that we process, process any child contents, so that we correctly
 455         # replay copies/replaces/etc.
 456         do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, prefix+"  ")
 457
 458 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
 459                    export_paths, prefix=""):
 460     # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
 461     # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
 462     #       associated remote repo folder. (Is this a problem?)
 463     paths_local =  get_svn_dirlist(path_offset)
 464     paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
 465     ui.status(prefix + ">> do_svn_add_dir: paths_local:  %s", str(paths_local),  level=ui.DEBUG, color='GREEN')
 466     ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
 467     # Update files/folders which exist in remote but not local
 468     for path in paths_remote:
 469         path_is_dir = True if path[-1] == "/" else False
 470         working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
 471         do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
 472                    export_paths, path_is_dir, prefix+"  ")
 473     # Remove files/folders which exist in local but not remote
 474     for path in paths_local:
 475         if not path in paths_remote:
 476             ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
 477             run_svn(["remove", "--force", path_offset+"/"+path])
 478             # TODO: Does this handle deleted folders too? Wouldn't want to have a case
 479             #       where we only delete all files from folder but leave orphaned folder around.
 480
 481 def process_svn_log_entry(log_entry, options, commit_paths, prefix = ""):
 482     """
 483     Process SVN changes from the given log entry. Build an array (commit_paths)
 484     of the paths in the working-copy that were changed, i.e. the paths which
 485     we'll pass to "svn commit".
 486     """
 487     export_paths = []
 488     source_rev = log_entry['revision']
 489     ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
 490     for d in log_entry['changed_paths']:
 491         # Get the full path for this changed_path
 492         # e.g. '/branches/bug123/projectA/file1.txt'
 493         path = d['path']
 494         if not is_child_path(path, source_base):
 495             # Ignore changed files that are not part of this subdir
 496             ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s  (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
 497             continue
 498         # Note: d['kind']="" for action="M" paths which only have property changes.
 499         if d['kind'] == "":
 500             d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
 501         assert (d['kind'] == 'file') or (d['kind'] == 'dir')
 502         path_is_dir =  True if d['kind'] == 'dir'  else False
 503         path_is_file = True if d['kind'] == 'file' else False
 504         # Calculate the offset (based on source_base) for this changed_path
 505         # e.g. 'projectA/file1.txt'
 506         # (path = source_base + "/" + path_offset)
 507         path_offset = path[len(source_base):].strip("/")
 508         # Get the action for this path
 509         action = d['action']
 510         if action not in _valid_svn_actions:
 511             raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 512                 % (source_rev, action))
 513         ui.status(" %s %s%s", action, d['path'],
 514             (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 515             level=ui.VERBOSE)
 516
 517         # Try to be efficient and keep track of an explicit list of paths in the
 518         # working copy that changed. If we commit from the root of the working copy,
 519         # then SVN needs to crawl the entire working copy looking for pending changes.
 520         commit_paths.append(path_offset)
 521
 522         # Special-handling for replace's
 523         if action == 'R':
 524             # If file was "replaced" (deleted then re-added, all in same revision),
 525             # then we need to run the "svn rm" first, then change action='A'. This
 526             # lets the normal code below handle re-"svn add"'ing the files. This
 527             # should replicate the "replace".
 528             run_svn(["remove", "--force", path_offset])
 529             action = 'A'
 530
 531         # Handle all the various action-types
 532         # (Handle "add" first, for "svn copy/move" support)
 533         if action == 'A':
 534             # Determine where to export from.
 535             svn_copy = False
 536             # Handle cases where this "add" was a copy from another URL in the source repo
 537             if d['copyfrom_revision']:
 538                 copyfrom_path = d['copyfrom_path']
 539                 copyfrom_rev =  d['copyfrom_revision']
 540                 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, prefix+"  ")
 541             # Else just "svn export" the files from the source repo and "svn add" them.
 542             else:
 543                 # Create (parent) directory if needed
 544                 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
 545                 if not os.path.exists(p_path):
 546                     run_svn(["mkdir", p_path])
 547                 # Export the entire added tree.
 548                 if path_is_dir:
 549                     # For directories, defer the (recurisve) "svn export". Might have a
 550                     # situation in a branch merge where the entry in the svn-log is a
 551                     # non-copy-from'd "add" but there are child contents (that we haven't
 552                     # gotten to yet in log_entry) that are copy-from's.  When we try do
 553                     # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
 554                     # having pre-existing (svn-add'd) contents creates some trouble.
 555                     # Instead, just create the stub folders ("svn mkdir" above) and defer
 556                     # exporting the final file-state until the end.
 557                     add_path(export_paths, path_offset)
 558                 else:
 559                     # Export the final verison of this file. We *need* to do this before running
 560                     # the "svn add", even if we end-up re-exporting this file again via export_paths.
 561                     run_svn(["export", "--force", "-r", source_rev,
 562                              source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 563                 if not in_svn(path_offset, prefix=prefix+"  "):
 564                     # Need to use in_svn here to handle cases where client committed the parent
 565                     # folder and each indiv sub-folder.
 566                     run_svn(["add", "--parents", path_offset])
 567                 # Copy SVN properties from source repo
 568                 sync_svn_props(source_url, source_rev, path_offset)
 569
 570         elif action == 'D':
 571             run_svn(["remove", "--force", path_offset])
 572
 573         elif action == 'M':
 574             if path_is_file:
 575                 run_svn(["export", "--force", "-N" , "-r", source_rev,
 576                          source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 577             # Update SVN properties based on source repo
 578             sync_svn_props(source_url, source_rev, path_offset)
 579
 580         else:
 581             raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
 582                 % action)
 583
 584     # Export the final version of all add'd paths from source_url
 585     if export_paths:
 586         for path_offset in export_paths:
 587             run_svn(["export", "--force", "-r", source_rev,
 588                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 589
 590 def disp_svn_log_summary(log_entry):
 591     ui.status("")
 592     ui.status("r%s | %s | %s",
 593         log_entry['revision'],
 594         log_entry['author'],
 595         str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
 596     ui.status(log_entry['message'])
 597     ui.status("------------------------------------------------------------------------")
 598
 599 def real_main(options, args):
 600     global source_url, target_url, rev_map
 601     source_url = args.pop(0).rstrip("/")    # e.g. 'http://server/svn/source/trunk'
 602     target_url = args.pop(0).rstrip("/")    # e.g. 'file:///svn/target/trunk'
 603     ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
 604
 605     # Make sure that both the source and target URL's are valid
 606     source_info = svnclient.get_svn_info(source_url)
 607     assert is_child_path(source_url, source_info['repos_url'])
 608     target_info = svnclient.get_svn_info(target_url)
 609     assert is_child_path(target_url, target_info['repos_url'])
 610
 611     # Init global vars
 612     global source_repos_url,source_base,source_repos_uuid
 613     source_repos_url = source_info['repos_url']       # e.g. 'http://server/svn/source'
 614     source_base = source_url[len(source_repos_url):]  # e.g. '/trunk'
 615     source_repos_uuid = source_info['repos_uuid']
 616
 617     # Init start and end revision
 618     source_start_rev = svnclient.get_svn_rev(source_repos_url, options.svn_rev_start if options.svn_rev_start else 1)
 619     source_end_rev   = svnclient.get_svn_rev(source_repos_url, options.svn_rev_end   if options.svn_rev_end   else "HEAD")
 620
 621     target_end_rev = target_info['revision']   # Last revision # in the target repo
 622     wc_target = os.path.abspath('_wc_target')
 623     num_entries_proc = 0
 624     commit_count = 0
 625     source_rev = None
 626     target_rev = None
 627
 628     # Check out a working copy of target_url if needed
 629     wc_exists = os.path.exists(wc_target)
 630     if wc_exists and not options.cont_from_break:
 631         shutil.rmtree(wc_target)
 632         wc_exists = False
 633     if not wc_exists:
 634         ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
 635         svnclient.svn_checkout(target_url, wc_target)
 636     os.chdir(wc_target)
 637
 638     if not options.cont_from_break:
 639         # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
 640         # Get the first log entry at/after source_start_rev, which is where
 641         # we'll do the initial import from.
 642         it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False)
 643         for source_start_log in it_log_start:
 644             break
 645         if not source_start_log:
 646             raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
 647                 (source_start_rev, source_end_rev, source_url))
 648
 649         # This is the revision we will start from for source_url
 650         source_start_rev = source_rev = int(source_start_log['revision'])
 651         ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
 652
 653         # For the initial commit to the target URL, export all the contents from
 654         # the source URL at the start-revision.
 655         disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
 656         ui.status("(Initial import)", level=ui.VERBOSE)
 657         # Export and add file-contents from source_url@source_start_rev
 658         top_paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
 659         top_paths = top_paths.strip("\n").split("\n")
 660         for path in top_paths:
 661             # For each top-level file/folder...
 662             if not path:
 663                 continue
 664             # Directories have a trailing slash in the "svn list" output
 665             path_is_dir = True if path[-1] == "/" else False
 666             path_offset = path.rstrip('/') if path_is_dir else path
 667             if in_svn(path_offset, prefix="  "):
 668                 raise InternalError("Cannot replay history on top of pre-existing structure: %s" % source_url+"/"+path_offset)
 669             if path_is_dir and not os.path.exists(path_offset):
 670                 os.makedirs(path_offset)
 671             run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 672             run_svn(["add", path_offset])
 673         # Update any properties on the newly added content
 674         paths = run_svn(["list", "--recursive", "-r", source_rev, source_url+"@"+str(source_rev)])
 675         paths = paths.strip("\n").split("\n")
 676         sync_svn_props(source_url, source_rev, "")
 677         for path in paths:
 678             if not path:
 679                 continue
 680             # Directories have a trailing slash in the "svn list" output
 681             path_is_dir = True if path[-1] == "/" else False
 682             path_offset = path.rstrip('/') if path_is_dir else path
 683             ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path_offset, level=ui.VERBOSE)
 684             sync_svn_props(source_url, source_rev, path_offset)
 685         # Commit the initial import
 686         num_entries_proc += 1
 687         target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 688         target_rev = commit_from_svn_log_entry(source_start_log, options, target_revprops=target_revprops)
 689         if target_rev:
 690             # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 691             set_rev_map(source_rev, target_rev)
 692             # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 693             run_svn(["update"])
 694             commit_count += 1
 695     else:
 696         # Re-build the rev_map based on any already-replayed history in target_url
 697         build_rev_map(target_url, target_end_rev, source_info)
 698         if not rev_map:
 699             raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url)
 700         source_start_rev = int(max(rev_map, key=rev_map.get))
 701         assert source_start_rev
 702         ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
 703
 704     svn_vers_t = svnclient.get_svn_client_version()
 705     svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
 706
 707     # Load SVN log starting from source_start_rev + 1
 708     it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else []
 709     source_rev = None
 710
 711     try:
 712         for log_entry in it_log_entries:
 713             if options.entries_proc_limit:
 714                 if num_entries_proc >= options.entries_proc_limit:
 715                     break
 716             # Replay this revision from source_url into target_url
 717             disp_svn_log_summary(log_entry)
 718             source_rev = log_entry['revision']
 719             # Process all the changed-paths in this log entry
 720             commit_paths = []
 721             process_svn_log_entry(log_entry, options, commit_paths)
 722             num_entries_proc += 1
 723             # Commit any changes made to _wc_target
 724             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 725             target_rev = commit_from_svn_log_entry(log_entry, options, commit_paths, target_revprops=target_revprops)
 726             if target_rev:
 727                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 728                 source_rev = log_entry['revision']
 729                 set_rev_map(source_rev, target_rev)
 730                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 731                 run_svn(["update"])
 732                 commit_count += 1
 733                 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
 734                 if svn_vers >= 1.7 and (commit_count % 100 == 0):
 735                     run_svn(["cleanup"])
 736         if not source_rev:
 737             # If there were no new source_url revisions to process, init source_rev
 738             # for the "finally" message below.
 739             source_rev = source_end_rev
 740
 741     except KeyboardInterrupt:
 742         print "\nStopped by user."
 743         print "\nCleaning-up..."
 744         run_svn(["cleanup"])
 745         full_svn_revert()
 746     except:
 747         print "\nCommand failed with following error:\n"
 748         traceback.print_exc()
 749         print "\nCleaning-up..."
 750         run_svn(["cleanup"])
 751         print run_svn(["status"])
 752         full_svn_revert()
 753     finally:
 754         print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
 755
 756 def main():
 757     # Defined as entry point. Must be callable without arguments.
 758     usage = "Usage: %prog [OPTIONS] source_url target_url"
 759     description = """\
 760   Replicate (replay) history from one SVN repository to another. Maintain
 761   logical ancestry wherever possible, so that 'svn log' on the replayed
 762   repo will correctly follow file/folder renames.
 763
 764   == Examples ==
 765   Create a copy of only /trunk from source repo, starting at r5000
 766   $ svnadmin create /svn/target
 767   $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
 768   $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
 769     1. The target_url will be checked-out to ./_wc_target
 770     2. The first commit to http://server/source/trunk at/after r5000 will be
 771        exported & added into _wc_target
 772     3. All revisions affecting http://server/source/trunk (starting at r5000)
 773        will be replayed to _wc_target. Any add/copy/move/replaces that are
 774        copy-from'd some path outside of /trunk (e.g. files renamed on a
 775        /branch and branch was merged into /trunk) will correctly maintain
 776        logical ancestry where possible.
 777
 778   Use continue-mode (-c) to pick-up where the last run left-off
 779   $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
 780     1. The target_url will be checked-out to ./_wc_target, if not already
 781        checked-out
 782     2. All new revisions affecting http://server/source/trunk starting from
 783        the last replayed revision to file:///svn/target/trunk (based on the
 784        svn2svn:* revprops) will be replayed to _wc_target, maintaining all
 785        logical ancestry where possible."""
 786     parser = optparse.OptionParser(usage, description=description,
 787                 formatter=HelpFormatter(), version="%prog "+str(full_version))
 788     #parser.remove_option("--help")
 789     #parser.add_option("-h", "--help", dest="show_help", action="store_true",
 790     #    help="show this help message and exit")
 791     parser.add_option("-r", "--revision", type="string", dest="svn_rev", metavar="ARG",
 792                       help="revision range to replay from source_url\n" + \
 793                            "A revision argument can be one of:\n" + \
 794                            "   START        start rev # (end will be 'HEAD')\n" + \
 795                            "   START:END    start and ending rev #'s\n" + \
 796                            "(Any revision # formats which SVN understands\n" + \
 797                            " are supported, e.g. 'HEAD', '{2010-01-31}', etc.)")
 798     parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", default=False,
 799                       help="maintain original 'Author' info from source repo")
 800     parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
 801                       help="continue from previous break")
 802     parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
 803                       help="maximum number of log entries to process")
 804     parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 805                       help="try processing next log entry but don't commit changes to "
 806                            "target working-copy (forces --limit=1)")
 807     parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
 808                       help="enable additional output (use -vv or -vvv for more)")
 809     parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
 810                       help="enable debugging output (same as -vvv)")
 811     options, args = parser.parse_args()
 812     if len(args) != 2:
 813         parser.error("incorrect number of arguments")
 814     if options.verbosity < 10:
 815         # Expand multiple "-v" arguments to a real ui._level value
 816         options.verbosity *= 10
 817     if options.dry_run:
 818         # When in dry-run mode, only try to process the next log_entry
 819         options.entries_proc_limit = 1
 820     options.svn_rev_start = None
 821     options.svn_rev_end   = None
 822     if options.svn_rev:
 823         rev = options.svn_rev.split(":")
 824         options.svn_rev_start = rev[0] if len(rev)>0 else None
 825         options.svn_rev_end   = rev[1] if len(rev)>1 else None
 826     ui.update_config(options)
 827     return real_main(options, args)
 828
 829
 830 if __name__ == "__main__":
 831     sys.exit(main() or 0)