svn2svn/run/svn2svn.py

   1 """
   2 Replicate (replay) changesets from one SVN repository to another.
   3 """
   4
   5 from .. import base_version, full_version
   6 from .. import ui
   7 from .. import svnclient
   8 from ..shell import run_svn
   9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
  10 from parse import HelpFormatter
  11
  12 import sys
  13 import os
  14 import time
  15 import traceback
  16 import shutil
  17 import operator
  18 import optparse
  19 from datetime import datetime
  20
  21 _valid_svn_actions = "MARD"   # The list of known SVN action abbr's, from "svn log"
  22
  23 # Module-level variables/parameters
  24 source_url = ""          # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
  25 source_repos_url = ""    # URL to root of source SVN repo,        e.g. 'http://server/svn/source'
  26 source_base = ""         # Relative path of source_url in source SVN repo, e.g. '/trunk'
  27 source_repos_uuid = ""   # UUID of source SVN repo
  28 target_url =""           # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
  29 rev_map = {}             # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
  30
  31 def commit_from_svn_log_entry(log_entry, options, commit_paths=None, target_revprops=None):
  32     """
  33     Given an SVN log entry and an optional list of changed paths, do an svn commit.
  34     """
  35     # TODO: Run optional external shell hook here, for doing pre-commit filtering
  36     # Display the _wc_target "svn status" info if running in -vv (or higher) mode
  37     if ui.get_level() >= ui.EXTRA:
  38         ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
  39         ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
  40     # This will use the local timezone for displaying commit times
  41     timestamp = int(log_entry['date'])
  42     svn_date = str(datetime.fromtimestamp(timestamp))
  43     # Uncomment this one one if you prefer UTC commit times
  44     #svn_date = "%d 0" % timestamp
  45     args = ["commit", "--force-log"]
  46     if options.keep_author:
  47         args += ["-m", log_entry['message'] + "\nDate: " + svn_date, "--username", log_entry['author']]
  48     else:
  49         args += ["-m", log_entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + log_entry['author']]
  50     revprops = {}
  51     if log_entry['revprops']:
  52         # Carry forward any revprop's from the source revision
  53         for v in log_entry['revprops']:
  54             revprops[v['name']] = v['value']
  55     if target_revprops:
  56         # Add any extra revprop's we want to set for the target repo commits
  57         for v in target_revprops:
  58             revprops[v['name']] = v['value']
  59     if revprops:
  60         for key in revprops:
  61             args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
  62     if commit_paths:
  63         if len(commit_paths)<100:
  64             # If we don't have an excessive amount of individual changed paths, pass
  65             # those to the "svn commit" command. Else, pass nothing so we commit at
  66             # the root of the working-copy.
  67             args += list(commit_paths)
  68     rev = None
  69     if not options.dry_run:
  70         # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
  71         output = run_svn(args)
  72         if output:
  73             output_lines = output.strip("\n").split("\n")
  74             rev = ""
  75             for line in output_lines:
  76                 if line[0:19] == 'Committed revision ':
  77                     rev = line[19:].rstrip('.')
  78                     break
  79             if rev:
  80                 ui.status("Committed revision %s.", rev)
  81     return rev
  82
  83 def full_svn_revert():
  84     """
  85     Do an "svn revert" and proactively remove any extra files in the working copy.
  86     """
  87     run_svn(["revert", "--recursive", "."])
  88     output = run_svn(["status"])
  89     if output:
  90         output_lines = output.strip("\n").split("\n")
  91         for line in output_lines:
  92             if line[0] == "?":
  93                 path = line[4:].strip(" ")
  94                 if os.path.isfile(path):
  95                     os.remove(path)
  96                 if os.path.isdir(path):
  97                     shutil.rmtree(path)
  98
  99 def gen_tracking_revprops(source_rev):
 100     """
 101     Build an array of svn2svn-specific source-tracking revprops.
 102     """
 103     revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
 104                 {'name':'svn2svn:source_url',  'value':source_url},
 105                 {'name':'svn2svn:source_rev',  'value':source_rev}]
 106     return revprops
 107
 108 def in_svn(p, require_in_repo=False, prefix=""):
 109     """
 110     Check if a given file/folder is being tracked by Subversion.
 111     Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
 112     With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
 113     Use "svn status" to check the status of the file/folder.
 114     """
 115     entries = svnclient.get_svn_status(p, no_recursive=True)
 116     if not entries:
 117         return False
 118     d = entries[0]
 119     if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
 120         # If caller requires this path to be in the SVN repo, prevent returning True
 121         # for paths that are only locally-added.
 122         ret = False
 123     else:
 124         # Don't consider files tracked as deleted in the WC as under source-control.
 125         # Consider files which are locally added/copied as under source-control.
 126         ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
 127     ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
 128     return ret
 129
 130 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
 131     """
 132     Given a source path, walk the SVN history backwards to inspect the ancestory of
 133     that path, seeing if it traces back to base_path.  Build an array of copyfrom_path
 134     and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
 135     which base_path is a substring match of (e.g. we crawled back to the initial branch-
 136     copy from trunk), then return the collection of ancestor paths.  Otherwise,
 137     copyfrom_path has no ancestory compared to base_path.
 138
 139     This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
 140     file/folder was renamed in a branch and then that branch was merged back to trunk.
 141
 142     'svn_repos_url' is the full URL to the root of the SVN repository,
 143       e.g. 'file:///path/to/repo'
 144     'base_path' is the path in the SVN repo to the target path we're trying to
 145       trace ancestry back to, e.g. 'trunk'.
 146     'source_path' is the path in the SVN repo to the source path to start checking
 147       ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
 148       (full_path = svn_repos_url+base_path+"/"+path_offset)
 149     'source_rev' is the revision to start walking the history of source_path backwards from.
 150     """
 151     ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s  base_path: %s",
 152         svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
 153     done = False
 154     working_path = base_path+"/"+source_path
 155     working_rev  = source_rev
 156     first_iter_done = False
 157     ancestors_temp = []
 158     while not done:
 159         # Get the first "svn log" entry for this path (relative to @rev)
 160         ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
 161         log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path+"@"+str(working_rev), 1, working_rev, True)
 162         if not log_entry:
 163             ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
 164             done = True
 165             break
 166         # If we found a copy-from case which matches our base_path, we're done.
 167         # ...but only if we've at least tried to search for the first copy-from path.
 168         if first_iter_done and working_path.startswith(base_path):
 169             ui.status(prefix + ">> find_svn_ancestors: Done: Found working_path.startswith(base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
 170             done = True
 171             break
 172         first_iter_done = True
 173         # Search for any actions on our target path (or parent paths).
 174         changed_paths_temp = []
 175         for d in log_entry['changed_paths']:
 176             path = d['path']
 177             if path in working_path:
 178                 changed_paths_temp.append({'path': path, 'data': d})
 179         if not changed_paths_temp:
 180             # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
 181             ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
 182             done = True
 183             continue
 184         # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
 185         changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
 186         # Find the action for our working_path in this revision. Use a loop to check in reverse order,
 187         # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
 188         for v in changed_paths:
 189             d = v['data']
 190             path = d['path']
 191             # Check action-type for this file
 192             action = d['action']
 193             if action not in _valid_svn_actions:
 194                 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 195                     % (log_entry['revision'], action))
 196             ui.status(prefix + "> %s %s%s", action, path,
 197                 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 198                 level=ui.DEBUG, color='YELLOW')
 199             if action == 'D':
 200                 # If file/folder was deleted, it has no ancestor
 201                 ancestors_temp = []
 202                 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
 203                 done = True
 204                 break
 205             if action in 'RA':
 206                 # If file/folder was added/replaced but not a copy, it has no ancestor
 207                 if not d['copyfrom_path']:
 208                     ancestors_temp = []
 209                     ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
 210                         "Added" if action == "A" else "Replaced",
 211                         level=ui.DEBUG, color='YELLOW')
 212                     done = True
 213                     break
 214                 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
 215                 # and keep checking for ancestors
 216                 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
 217                     action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
 218                     level=ui.DEBUG, color='YELLOW')
 219                 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
 220                                        'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
 221                 working_path = working_path.replace(d['path'], d['copyfrom_path'])
 222                 working_rev =  d['copyfrom_revision']
 223                 # Follow the copy and keep on searching
 224                 break
 225     ancestors = []
 226     if ancestors_temp:
 227         ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
 228         working_path = base_path+"/"+source_path
 229         for idx in range(len(ancestors_temp)):
 230             d = ancestors_temp[idx]
 231             working_path = working_path.replace(d['path'], d['copyfrom_path'])
 232             working_rev =  d['copyfrom_rev']
 233             ancestors.append({'path': working_path, 'revision': working_rev})
 234         if ui.get_level() >= ui.DEBUG:
 235             max_len = 0
 236             for idx in range(len(ancestors)):
 237                 d = ancestors[idx]
 238                 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
 239             ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
 240             for idx in range(len(ancestors)-1):
 241                 d = ancestors[idx]
 242                 d_next = ancestors[idx+1]
 243                 ui.status(prefix + " [%s] %s <-- %s", idx,
 244                     str(d['path']+"@"+str(d['revision'])).ljust(max_len),
 245                     str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
 246                     level=ui.DEBUG, color='YELLOW')
 247     else:
 248         ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
 249             svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
 250     return ancestors
 251
 252 def get_rev_map(source_rev, prefix):
 253     """
 254     Find the equivalent rev # in the target repo for the given rev # from the source repo.
 255     """
 256     ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
 257     # Find the highest entry less-than-or-equal-to source_rev
 258     for rev in range(int(source_rev), 0, -1):
 259         ui.status(prefix + ">> get_rev_map: rev=%s  in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
 260         if rev in rev_map:
 261             return int(rev_map[rev])
 262     # Else, we fell off the bottom of the rev_map. Ruh-roh...
 263     return None
 264
 265 def set_rev_map(source_rev, target_rev):
 266     ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
 267     global rev_map
 268     rev_map[int(source_rev)]=int(target_rev)
 269
 270 def build_rev_map(target_url, source_info):
 271     """
 272     Check for any already-replayed history from source_url (source_info) and
 273     build the mapping-table of source_rev -> target_rev.
 274     """
 275     global rev_map
 276     rev_map = {}
 277     ui.status("Rebuilding rev_map...", level=ui.VERBOSE)
 278     proc_count = 0
 279     it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, 'HEAD', get_changed_paths=False, get_revprops=True)
 280     for log_entry in it_log_entries:
 281         if log_entry['revprops']:
 282             revprops = {}
 283             for v in log_entry['revprops']:
 284                 if v['name'].startswith('svn2svn:'):
 285                     revprops[v['name']] = v['value']
 286             if revprops and \
 287                revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
 288                revprops['svn2svn:source_url'] == source_info['url']:
 289                 source_rev = revprops['svn2svn:source_rev']
 290                 target_rev = log_entry['revision']
 291                 set_rev_map(source_rev, target_rev)
 292
 293 def get_svn_dirlist(svn_path, svn_rev = ""):
 294     """
 295     Get a list of all the child contents (recusive) of the given folder path.
 296     """
 297     args = ["list"]
 298     path = svn_path
 299     if svn_rev:
 300         args += ["-r", svn_rev]
 301         path += "@"+str(svn_rev)
 302     args += [path]
 303     paths = run_svn(args, no_fail=True)
 304     paths = paths.strip("\n").split("\n") if len(paths)>1 else []
 305     return paths
 306
 307 def path_in_list(paths, path):
 308     for p in paths:
 309         if path.startswith(p):
 310             return True
 311     return False
 312
 313 def add_path(paths, path):
 314     if not path_in_list(paths, path):
 315         paths.append(path)
 316
 317 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
 318                export_paths={}, is_dir = False, prefix = ""):
 319     """
 320     Given the add'd source path, replay the "svn add/copy" commands to correctly
 321     track renames across copy-from's.
 322
 323     For example, consider a sequence of events like this:
 324     1. svn copy /trunk /branches/fix1
 325     2. (Make some changes on /branches/fix1)
 326     3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2  " Rename folder
 327     4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt  " Rename file inside renamed folder
 328     5. svn co /trunk && svn merge /branches/fix1
 329     After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
 330     and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
 331     to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
 332     we'd lose the logical history that Proj2/file2.txt is really a descendant
 333     of Proj1/file1.txt.
 334
 335     'path_offset' is the offset from source_base to the file to check ancestry for,
 336       e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
 337     'source_rev' is the revision ("svn log") that we're processing from the source repo.
 338     'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
 339       directory, when being called recursively by do_svn_add_dir().
 340     'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
 341     'is_dir' is whether path_offset is a directory (rather than a file).
 342     """
 343     ui.status(prefix + ">> do_svn_add: %s  %s", source_base+"/"+path_offset+"@"+str(source_rev),
 344         "  (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
 345         level=ui.DEBUG, color='GREEN')
 346     # Check if the given path has ancestors which chain back to the current source_base
 347     found_ancestor = False
 348     ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+"  ")
 349     # ancestors[n] is the original (pre-branch-copy) trunk path.
 350     # ancestors[n-1] is the first commit on the new branch.
 351     copyfrom_path = ancestors[len(ancestors)-1]['path']     if ancestors else ""
 352     copyfrom_rev  = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
 353     if ancestors:
 354         # The copy-from path has ancestory back to source_url.
 355         ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
 356             level=ui.DEBUG, color='GREEN', bold=True)
 357         found_ancestor = True
 358         # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
 359         # return None in the case where copyfrom_rev is *before* our source_start_rev.
 360         tgt_rev = get_rev_map(copyfrom_rev, prefix+"  ")
 361         ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
 362     else:
 363         ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
 364         found_ancestor = False
 365     if found_ancestor and tgt_rev:
 366         # Check if this path_offset in the target WC already has this ancestry, in which
 367         # case there's no need to run the "svn copy" (again).
 368         path_in_svn = in_svn(path_offset, prefix=prefix+"  ")
 369         log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+"  ") else []
 370         if (not log_entry or (log_entry['revision'] != tgt_rev)):
 371             copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
 372             ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 373             ui.status(prefix + "   copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 374             ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
 375             if path_in_svn and \
 376                ((parent_copyfrom_path and copyfrom_path.startswith(parent_copyfrom_path)) and \
 377                 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
 378                 # When being called recursively, if this child entry has the same ancestor as the
 379                 # the parent, then no need to try to run another "svn copy".
 380                 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
 381                     parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
 382                 pass
 383             else:
 384                 # Copy this path from the equivalent path+rev in the target repo, to create the
 385                 # equivalent history.
 386                 if parent_copyfrom_path:
 387                     # If we have a parent copy-from path, we mis-match that so display a status
 388                     # message describing the action we're mimic'ing. If path_in_svn, then this
 389                     # is logically a "replace" rather than an "add".
 390                     ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
 391                 if path_in_svn:
 392                     # If local file is already under version-control, then this is a replace.
 393                     ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
 394                     run_svn(["remove", "--force", path_offset])
 395                 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
 396                 # Export the final version of this file/folder from the source repo, to make
 397                 # sure we're up-to-date.
 398                 add_path(export_paths, path_offset)
 399         else:
 400             ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
 401     else:
 402         # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
 403         # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
 404         # Create (parent) directory if needed.
 405         # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
 406         #       split-out to a shared tag?
 407         p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
 408         if not os.path.exists(p_path):
 409             run_svn(["mkdir", p_path])
 410         if not in_svn(path_offset, prefix=prefix+"  "):
 411             if is_dir:
 412                 # Export the final verison of all files in this folder.
 413                 add_path(export_paths, path_offset)
 414             else:
 415                 # Export the final verison of this file. We *need* to do this before running
 416                 # the "svn add", even if we end-up re-exporting this file again via export_paths.
 417                 run_svn(["export", "--force", "-r", source_rev,
 418                          source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 419             # If not already under version-control, then "svn add" this file/folder.
 420             run_svn(["add", "--parents", path_offset])
 421         # TODO: Need to copy SVN properties from source repos
 422     if is_dir:
 423         # For any folders that we process, process any child contents, so that we correctly
 424         # replay copies/replaces/etc.
 425         do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, prefix+"  ")
 426
 427 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
 428                    export_paths, prefix=""):
 429     # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
 430     # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
 431     #       associated remote repo folder. (Is this a problem?)
 432     paths_local =  get_svn_dirlist(path_offset)
 433     paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
 434     ui.status(prefix + ">> do_svn_add_dir: paths_local:  %s", str(paths_local),  level=ui.DEBUG, color='GREEN')
 435     ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
 436     # Update files/folders which exist in remote but not local
 437     for path in paths_remote:
 438         path_is_dir = True if path[-1] == "/" else False
 439         working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
 440         do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
 441                    export_paths, path_is_dir, prefix+"  ")
 442     # Remove files/folders which exist in local but not remote
 443     for path in paths_local:
 444         if not path in paths_remote:
 445             ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
 446             run_svn(["remove", "--force", path_offset+"/"+path])
 447             # TODO: Does this handle deleted folders too? Wouldn't want to have a case
 448             #       where we only delete all files from folder but leave orphaned folder around.
 449
 450 def process_svn_log_entry(log_entry, options, commit_paths, prefix = ""):
 451     """
 452     Process SVN changes from the given log entry. Build an array (commit_paths)
 453     of the paths in the working-copy that were changed, i.e. the paths which
 454     we'll pass to "svn commit".
 455     """
 456     export_paths = []
 457     source_rev = log_entry['revision']
 458     ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
 459     for d in log_entry['changed_paths']:
 460         # Get the full path for this changed_path
 461         # e.g. '/branches/bug123/projectA/file1.txt'
 462         path = d['path']
 463         if not path.startswith(source_base + "/"):
 464             # Ignore changed files that are not part of this subdir
 465             if path != source_base:
 466                 ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s  (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
 467             continue
 468         # Note: d['kind']="" for action="M" paths which only have property changes.
 469         path_is_dir =  True if d['kind'] == 'dir'  else False
 470         path_is_file = True if d['kind'] == 'file' else False
 471         # Calculate the offset (based on source_base) for this changed_path
 472         # e.g. 'projectA/file1.txt'
 473         # (path = source_base + "/" + path_offset)
 474         path_offset = path[len(source_base):].strip("/")
 475         # Get the action for this path
 476         action = d['action']
 477         if action not in _valid_svn_actions:
 478             raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 479                 % (source_rev, action))
 480         ui.status(" %s %s%s", action, d['path'],
 481             (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 482             level=ui.VERBOSE)
 483
 484         # Try to be efficient and keep track of an explicit list of paths in the
 485         # working copy that changed. If we commit from the root of the working copy,
 486         # then SVN needs to crawl the entire working copy looking for pending changes.
 487         add_path(commit_paths, path_offset)
 488
 489         # Special-handling for replace's
 490         if action == 'R':
 491             # If file was "replaced" (deleted then re-added, all in same revision),
 492             # then we need to run the "svn rm" first, then change action='A'. This
 493             # lets the normal code below handle re-"svn add"'ing the files. This
 494             # should replicate the "replace".
 495             run_svn(["remove", "--force", path_offset])
 496             action = 'A'
 497
 498         # Handle all the various action-types
 499         # (Handle "add" first, for "svn copy/move" support)
 500         if action == 'A':
 501             # Determine where to export from.
 502             svn_copy = False
 503             # Handle cases where this "add" was a copy from another URL in the source repos
 504             if d['copyfrom_revision']:
 505                 copyfrom_path = d['copyfrom_path']
 506                 copyfrom_rev =  d['copyfrom_revision']
 507                 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, prefix+"  ")
 508             # Else just "svn export" the files from the source repo and "svn add" them.
 509             else:
 510                 # Create (parent) directory if needed
 511                 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
 512                 if not os.path.exists(p_path):
 513                     run_svn(["mkdir", p_path])
 514                 # Export the entire added tree.
 515                 if path_is_dir:
 516                     # For directories, defer the (recurisve) "svn export". Might have a
 517                     # situation in a branch merge where the entry in the svn-log is a
 518                     # non-copy-from'd "add" but there are child contents (that we haven't
 519                     # gotten to yet in log_entry) that are copy-from's.  When we try do
 520                     # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
 521                     # having pre-existing (svn-add'd) contents creates some trouble.
 522                     # Instead, just create the stub folders ("svn mkdir" above) and defer
 523                     # exporting the final file-state until the end.
 524                     add_path(export_paths, path_offset)
 525                 else:
 526                     # Export the final verison of this file. We *need* to do this before running
 527                     # the "svn add", even if we end-up re-exporting this file again via export_paths.
 528                     run_svn(["export", "--force", "-r", source_rev,
 529                              source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 530                 if not in_svn(path_offset, prefix=prefix+"  "):
 531                     # Need to use in_svn here to handle cases where client committed the parent
 532                     # folder and each indiv sub-folder.
 533                     run_svn(["add", "--parents", path_offset])
 534                 # TODO: Need to copy SVN properties from source repos
 535
 536         elif action == 'D':
 537             run_svn(["remove", "--force", path_offset])
 538
 539         elif action == 'M':
 540             # TODO: Is "svn merge -c" correct here? Should this just be an "svn export" plus
 541             #       proplist updating?
 542             out = run_svn(["merge", "-c", source_rev, "--non-recursive",
 543                      "--non-interactive", "--accept=theirs-full",
 544                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 545             # TODO: If d['props'] == 'modified', then run code to clean-up/purge any newly-modified props?
 546
 547         else:
 548             raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
 549                 % action)
 550
 551     # Export the final version of all add'd paths from source_url
 552     if export_paths:
 553         for path_offset in export_paths:
 554             run_svn(["export", "--force", "-r", source_rev,
 555                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 556
 557 def disp_svn_log_summary(log_entry):
 558     ui.status("")
 559     ui.status("r%s | %s | %s",
 560         log_entry['revision'],
 561         log_entry['author'],
 562         str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
 563     ui.status(log_entry['message'])
 564     ui.status("------------------------------------------------------------------------")
 565
 566 def real_main(options, args):
 567     global source_url, target_url, rev_map
 568     source_url = args.pop(0).rstrip("/")    # e.g. 'http://server/svn/source/trunk'
 569     target_url = args.pop(0).rstrip("/")    # e.g. 'file:///svn/target/trunk'
 570     ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
 571
 572     # Make sure that both the source and target URL's are valid
 573     source_info = svnclient.get_svn_info(source_url)
 574     assert source_url.startswith(source_info['repos_url'])
 575     target_info = svnclient.get_svn_info(target_url)
 576     assert target_url.startswith(target_info['repos_url'])
 577
 578     # Init global vars
 579     global source_repos_url,source_base,source_repos_uuid
 580     source_repos_url = source_info['repos_url']       # e.g. 'http://server/svn/source'
 581     source_base = source_url[len(source_repos_url):]  # e.g. '/trunk'
 582     source_repos_uuid = source_info['repos_uuid']
 583
 584     source_end_rev = source_info['revision']   # Last revision # in the source repo
 585     wc_target = os.path.abspath('_wc_target')
 586     num_entries_proc = 0
 587     commit_count = 0
 588     source_rev = None
 589     target_rev = None
 590
 591     # Check out a working copy of target_url if needed
 592     wc_exists = os.path.exists(wc_target)
 593     if wc_exists and not options.cont_from_break:
 594         shutil.rmtree(wc_target)
 595         wc_exists = False
 596     if not wc_exists:
 597         svnclient.svn_checkout(target_url, wc_target)
 598     os.chdir(wc_target)
 599
 600     if not options.cont_from_break:
 601         # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
 602         # Get log entry for the SVN revision we will check out
 603         if options.svn_rev:
 604             # If specify a rev, get log entry just before or at rev
 605             source_start_log = svnclient.get_last_svn_log_entry(source_url, 1, options.svn_rev, False)
 606         else:
 607             # Otherwise, get log entry of branch creation
 608             # Note: Trying to use svnclient.get_first_svn_log_entry(source_url, 1, source_end_rev, False)
 609             # ends-up being *VERY* time-consuming on a repo with lots of revisions. Even though
 610             # the "svn log" call is passing --limit 1, it seems like that limit-filter is happening
 611             # _after_ svn has fetched the full log history. Instead, search the history in chunks
 612             # and write some progress to the screen.
 613             ui.status("Searching for start source revision (%s)...", source_url, level=ui.VERBOSE)
 614             rev = 1
 615             chunk_size = 1000
 616             done = False
 617             while not done:
 618                 entries = svnclient.run_svn_log(source_url, rev, min(rev+chunk_size-1, target_info['revision']), 1, get_changed_paths=False)
 619                 if entries:
 620                     source_start_log = entries[0]
 621                     done = True
 622                     break
 623                 ui.status("...%s...", rev)
 624                 rev = rev+chunk_size
 625                 if rev > target_info['revision']:
 626                     done = True
 627             if not source_start_log:
 628                 raise InternalError("Unable to find first revision for source_url: %s" % source_url)
 629
 630         # This is the revision we will start from for source_url
 631         source_start_rev = source_rev = int(source_start_log['revision'])
 632         ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
 633
 634         # For the initial commit to the target URL, export all the contents from
 635         # the source URL at the start-revision.
 636         paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
 637         if len(paths)>1:
 638             disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
 639             ui.status("(Initial import)", level=ui.VERBOSE)
 640             paths = paths.strip("\n").split("\n")
 641             for path_raw in paths:
 642                 # For each top-level file/folder...
 643                 if not path_raw:
 644                     continue
 645                 # Directories have a trailing slash in the "svn list" output
 646                 path_is_dir = True if path_raw[-1] == "/" else False
 647                 path = path_raw.rstrip('/') if path_is_dir else path_raw
 648                 if path_is_dir and not os.path.exists(path):
 649                     os.makedirs(path)
 650                 ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path, level=ui.VERBOSE)
 651                 run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path+"@"+str(source_rev), path])
 652                 run_svn(["add", path])
 653             num_entries_proc += 1
 654             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 655             target_rev = commit_from_svn_log_entry(source_start_log, options, target_revprops=target_revprops)
 656             if target_rev:
 657                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 658                 set_rev_map(source_rev, target_rev)
 659                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 660                 run_svn(["update"])
 661                 commit_count += 1
 662     else:
 663         # Re-build the rev_map based on any already-replayed history in target_url
 664         build_rev_map(target_url, source_info)
 665         if not rev_map:
 666             raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url)
 667         source_start_rev = int(max(rev_map, key=rev_map.get))
 668         assert source_start_rev
 669         ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
 670
 671     svn_vers_t = svnclient.get_svn_client_version()
 672     svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
 673
 674     # Load SVN log starting from source_start_rev + 1
 675     it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True)
 676     source_rev = None
 677
 678     try:
 679         for log_entry in it_log_entries:
 680             if options.entries_proc_limit:
 681                 if num_entries_proc >= options.entries_proc_limit:
 682                     break
 683             # Replay this revision from source_url into target_url
 684             disp_svn_log_summary(log_entry)
 685             source_rev = log_entry['revision']
 686             # Process all the changed-paths in this log entry
 687             commit_paths = []
 688             process_svn_log_entry(log_entry, options, commit_paths)
 689             num_entries_proc += 1
 690             # Commit any changes made to _wc_target
 691             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 692             target_rev = commit_from_svn_log_entry(log_entry, options, commit_paths, target_revprops=target_revprops)
 693             if target_rev:
 694                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 695                 source_rev = log_entry['revision']
 696                 set_rev_map(source_rev, target_rev)
 697                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 698                 run_svn(["update"])
 699                 commit_count += 1
 700                 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
 701                 if svn_vers >= 1.7 and (commit_count % 100 == 0):
 702                     run_svn(["cleanup"])
 703         if not source_rev:
 704             # If there were no new source_url revisions to process, init source_rev
 705             # for the "finally" message below.
 706             source_rev = source_end_rev
 707
 708     except KeyboardInterrupt:
 709         print "\nStopped by user."
 710         print "\nCleaning-up..."
 711         run_svn(["cleanup"])
 712         full_svn_revert()
 713     except:
 714         print "\nCommand failed with following error:\n"
 715         traceback.print_exc()
 716         print "\nCleaning-up..."
 717         run_svn(["cleanup"])
 718         print run_svn(["status"])
 719         full_svn_revert()
 720     finally:
 721         print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
 722
 723 def main():
 724     # Defined as entry point. Must be callable without arguments.
 725     usage = "Usage: %prog [OPTIONS] source_url target_url"
 726     description = """\
 727   Replicate (replay) history from one SVN repository to another. Maintain
 728   logical ancestry wherever possible, so that 'svn log' on the replayed
 729   repo will correctly follow file/folder renames.
 730
 731   == Examples ==
 732   Create a copy of only /trunk from source repo, starting at r5000
 733   $ svnadmin create /svn/target
 734   $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
 735   $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
 736     1. The target_url will be checked-out to ./_wc_target
 737     2. The first commit to http://server/source/trunk at/after r5000 will be
 738        exported & added into _wc_target
 739     3. All revisions affecting http://server/source/trunk (starting at r5000)
 740        will be replayed to _wc_target. Any add/copy/move/replaces that are
 741        copy-from'd some path outside of /trunk (e.g. files renamed on a
 742        /branch and branch was merged into /trunk) will correctly maintain
 743        logical ancestry where possible.
 744
 745   Use continue-mode (-c) to pick-up where the last run left-off
 746   $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
 747     1. The target_url will be checked-out to ./_wc_target, if not already
 748        checked-out
 749     2. All new revisions affecting http://server/source/trunk starting from
 750        the last replayed revision to file:///svn/target/trunk (based on the
 751        svn2svn:* revprops) will be replayed to _wc_target, maintaining all
 752        logical ancestry where possible."""
 753     parser = optparse.OptionParser(usage, description=description,
 754                 formatter=HelpFormatter(), version="%prog "+str(full_version))
 755     #parser.remove_option("--help")
 756     #parser.add_option("-h", "--help", dest="show_help", action="store_true",
 757     #    help="show this help message and exit")
 758     parser.add_option("-r", "--revision", type="int", dest="svn_rev", metavar="REV",
 759                       help="initial SVN revision to start source_url replay")
 760     parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", default=False,
 761                       help="maintain original 'Author' info from source repo")
 762     parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
 763                       help="continue from previous break")
 764     parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
 765                       help="maximum number of log entries to process")
 766     parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 767                       help="try processing next log entry but don't commit changes to "
 768                            "target working-copy (forces --limit=1)")
 769     parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
 770                       help="enable additional output (use -vv or -vvv for more)")
 771     parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
 772                       help="enable debugging output (same as -vvv)")
 773     options, args = parser.parse_args()
 774     if len(args) != 2:
 775         parser.error("incorrect number of arguments")
 776     if options.verbosity < 10:
 777         # Expand multiple "-v" arguments to a real ui._level value
 778         options.verbosity *= 10
 779     if options.dry_run:
 780         # When in dry-run mode, only try to process the next log_entry
 781         options.entries_proc_limit = 1
 782     ui.update_config(options)
 783     return real_main(options, args)
 784
 785
 786 if __name__ == "__main__":
 787     sys.exit(main() or 0)