svn2svn/run/svn2svn.py

   1 """
   2 Replicate (replay) changesets from one SVN repository to another.
   3 """
   4
   5 from .. import base_version, full_version
   6 from .. import ui
   7 from .. import svnclient
   8 from ..shell import run_svn
   9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
  10 from parse import HelpFormatter
  11
  12 import sys
  13 import os
  14 import time
  15 import traceback
  16 import shutil
  17 import operator
  18 import optparse
  19 from datetime import datetime
  20
  21 _valid_svn_actions = "MARD"   # The list of known SVN action abbr's, from "svn log"
  22
  23 # Module-level variables/parameters
  24 source_url = ""          # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
  25 source_repos_url = ""    # URL to root of source SVN repo,        e.g. 'http://server/svn/source'
  26 source_base = ""         # Relative path of source_url in source SVN repo, e.g. '/trunk'
  27 source_repos_uuid = ""   # UUID of source SVN repo
  28 target_url =""           # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
  29 rev_map = {}             # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
  30
  31 def commit_from_svn_log_entry(log_entry, options, commit_paths=None, target_revprops=None):
  32     """
  33     Given an SVN log entry and an optional list of changed paths, do an svn commit.
  34     """
  35     # TODO: Run optional external shell hook here, for doing pre-commit filtering
  36     # Display the _wc_target "svn status" info if running in -vv (or higher) mode
  37     if ui.get_level() >= ui.EXTRA:
  38         ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
  39         ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
  40     # This will use the local timezone for displaying commit times
  41     timestamp = int(log_entry['date'])
  42     svn_date = str(datetime.fromtimestamp(timestamp))
  43     # Uncomment this one one if you prefer UTC commit times
  44     #svn_date = "%d 0" % timestamp
  45     args = ["commit", "--force-log"]
  46     if options.keep_author:
  47         args += ["-m", log_entry['message'] + "\nDate: " + svn_date, "--username", log_entry['author']]
  48     else:
  49         args += ["-m", log_entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + log_entry['author']]
  50     revprops = {}
  51     if log_entry['revprops']:
  52         # Carry forward any revprop's from the source revision
  53         for v in log_entry['revprops']:
  54             revprops[v['name']] = v['value']
  55     if target_revprops:
  56         # Add any extra revprop's we want to set for the target repo commits
  57         for v in target_revprops:
  58             revprops[v['name']] = v['value']
  59     if revprops:
  60         for key in revprops:
  61             args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
  62     if commit_paths:
  63         if len(commit_paths)<100:
  64             # If we don't have an excessive amount of individual changed paths, pass
  65             # those to the "svn commit" command. Else, pass nothing so we commit at
  66             # the root of the working-copy.
  67             args += list(commit_paths)
  68     rev = None
  69     if not options.dry_run:
  70         # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
  71         output = run_svn(args)
  72         if output:
  73             output_lines = output.strip("\n").split("\n")
  74             rev = ""
  75             for line in output_lines:
  76                 if line[0:19] == 'Committed revision ':
  77                     rev = line[19:].rstrip('.')
  78                     break
  79             if rev:
  80                 ui.status("Committed revision %s.", rev)
  81     return rev
  82
  83 def full_svn_revert():
  84     """
  85     Do an "svn revert" and proactively remove any extra files in the working copy.
  86     """
  87     run_svn(["revert", "--recursive", "."])
  88     output = run_svn(["status"])
  89     if output:
  90         output_lines = output.strip("\n").split("\n")
  91         for line in output_lines:
  92             if line[0] == "?":
  93                 path = line[4:].strip(" ")
  94                 if os.path.isfile(path):
  95                     os.remove(path)
  96                 if os.path.isdir(path):
  97                     shutil.rmtree(path)
  98
  99 def gen_tracking_revprops(source_rev):
 100     """
 101     Build an array of svn2svn-specific source-tracking revprops.
 102     """
 103     revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
 104                 {'name':'svn2svn:source_url',  'value':source_url},
 105                 {'name':'svn2svn:source_rev',  'value':source_rev}]
 106     return revprops
 107
 108 def in_svn(p, require_in_repo=False, prefix=""):
 109     """
 110     Check if a given file/folder is being tracked by Subversion.
 111     Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
 112     With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
 113     Use "svn status" to check the status of the file/folder.
 114     """
 115     entries = svnclient.get_svn_status(p, no_recursive=True)
 116     if not entries:
 117         return False
 118     d = entries[0]
 119     if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
 120         # If caller requires this path to be in the SVN repo, prevent returning True
 121         # for paths that are only locally-added.
 122         ret = False
 123     else:
 124         # Don't consider files tracked as deleted in the WC as under source-control.
 125         # Consider files which are locally added/copied as under source-control.
 126         ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
 127     ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
 128     return ret
 129
 130 def is_child_path(path, p_path):
 131     return True if (path == p_path) or (path.startswith(p_path+"/")) else False
 132
 133 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
 134     """
 135     Given a source path, walk the SVN history backwards to inspect the ancestory of
 136     that path, seeing if it traces back to base_path.  Build an array of copyfrom_path
 137     and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
 138     which base_path is a substring match of (e.g. we crawled back to the initial branch-
 139     copy from trunk), then return the collection of ancestor paths.  Otherwise,
 140     copyfrom_path has no ancestory compared to base_path.
 141
 142     This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
 143     file/folder was renamed in a branch and then that branch was merged back to trunk.
 144
 145     'svn_repos_url' is the full URL to the root of the SVN repository,
 146       e.g. 'file:///path/to/repo'
 147     'base_path' is the path in the SVN repo to the target path we're trying to
 148       trace ancestry back to, e.g. 'trunk'.
 149     'source_path' is the path in the SVN repo to the source path to start checking
 150       ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
 151       (full_path = svn_repos_url+base_path+"/"+path_offset)
 152     'source_rev' is the revision to start walking the history of source_path backwards from.
 153     """
 154     ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s  base_path: %s",
 155         svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
 156     done = False
 157     working_path = base_path+"/"+source_path
 158     working_rev  = source_rev
 159     first_iter_done = False
 160     ancestors_temp = []
 161     while not done:
 162         # Get the first "svn log" entry for this path (relative to @rev)
 163         ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
 164         log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path, 1, working_rev, True)
 165         if not log_entry:
 166             ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
 167             done = True
 168             break
 169         # If we found a copy-from case which matches our base_path, we're done.
 170         # ...but only if we've at least tried to search for the first copy-from path.
 171         if first_iter_done and is_child_path(working_path, base_path):
 172             ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(working_path, base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
 173             done = True
 174             break
 175         first_iter_done = True
 176         # Search for any actions on our target path (or parent paths).
 177         changed_paths_temp = []
 178         for d in log_entry['changed_paths']:
 179             path = d['path']
 180             if path in working_path:
 181                 changed_paths_temp.append({'path': path, 'data': d})
 182         if not changed_paths_temp:
 183             # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
 184             ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
 185             done = True
 186             continue
 187         # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
 188         changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
 189         # Find the action for our working_path in this revision. Use a loop to check in reverse order,
 190         # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
 191         for v in changed_paths:
 192             d = v['data']
 193             path = d['path']
 194             # Check action-type for this file
 195             action = d['action']
 196             if action not in _valid_svn_actions:
 197                 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 198                     % (log_entry['revision'], action))
 199             ui.status(prefix + "> %s %s%s", action, path,
 200                 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 201                 level=ui.DEBUG, color='YELLOW')
 202             if action == 'D':
 203                 # If file/folder was deleted, it has no ancestor
 204                 ancestors_temp = []
 205                 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
 206                 done = True
 207                 break
 208             if action in 'RA':
 209                 # If file/folder was added/replaced but not a copy, it has no ancestor
 210                 if not d['copyfrom_path']:
 211                     ancestors_temp = []
 212                     ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
 213                         "Added" if action == "A" else "Replaced",
 214                         level=ui.DEBUG, color='YELLOW')
 215                     done = True
 216                     break
 217                 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
 218                 # and keep checking for ancestors
 219                 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
 220                     action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
 221                     level=ui.DEBUG, color='YELLOW')
 222                 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
 223                                        'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
 224                 working_path = working_path.replace(d['path'], d['copyfrom_path'])
 225                 working_rev =  d['copyfrom_revision']
 226                 # Follow the copy and keep on searching
 227                 break
 228     ancestors = []
 229     if ancestors_temp:
 230         ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
 231         working_path = base_path+"/"+source_path
 232         for idx in range(len(ancestors_temp)):
 233             d = ancestors_temp[idx]
 234             working_path = working_path.replace(d['path'], d['copyfrom_path'])
 235             working_rev =  d['copyfrom_rev']
 236             ancestors.append({'path': working_path, 'revision': working_rev})
 237         if ui.get_level() >= ui.DEBUG:
 238             max_len = 0
 239             for idx in range(len(ancestors)):
 240                 d = ancestors[idx]
 241                 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
 242             ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
 243             for idx in range(len(ancestors)-1):
 244                 d = ancestors[idx]
 245                 d_next = ancestors[idx+1]
 246                 ui.status(prefix + " [%s] %s <-- %s", idx,
 247                     str(d['path']+"@"+str(d['revision'])).ljust(max_len),
 248                     str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
 249                     level=ui.DEBUG, color='YELLOW')
 250     else:
 251         ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
 252             svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
 253     return ancestors
 254
 255 def get_rev_map(source_rev, prefix):
 256     """
 257     Find the equivalent rev # in the target repo for the given rev # from the source repo.
 258     """
 259     ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
 260     # Find the highest entry less-than-or-equal-to source_rev
 261     for rev in range(int(source_rev), 0, -1):
 262         ui.status(prefix + ">> get_rev_map: rev=%s  in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
 263         if rev in rev_map:
 264             return int(rev_map[rev])
 265     # Else, we fell off the bottom of the rev_map. Ruh-roh...
 266     return None
 267
 268 def set_rev_map(source_rev, target_rev):
 269     ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
 270     global rev_map
 271     rev_map[int(source_rev)]=int(target_rev)
 272
 273 def build_rev_map(target_url, target_end_rev, source_info):
 274     """
 275     Check for any already-replayed history from source_url (source_info) and
 276     build the mapping-table of source_rev -> target_rev.
 277     """
 278     global rev_map
 279     rev_map = {}
 280     ui.status("Rebuilding rev_map...", level=ui.VERBOSE)
 281     proc_count = 0
 282     it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
 283     for log_entry in it_log_entries:
 284         if log_entry['revprops']:
 285             revprops = {}
 286             for v in log_entry['revprops']:
 287                 if v['name'].startswith('svn2svn:'):
 288                     revprops[v['name']] = v['value']
 289             if revprops and \
 290                revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
 291                revprops['svn2svn:source_url'] == source_info['url']:
 292                 source_rev = revprops['svn2svn:source_rev']
 293                 target_rev = log_entry['revision']
 294                 set_rev_map(source_rev, target_rev)
 295
 296 def get_svn_dirlist(svn_path, svn_rev = ""):
 297     """
 298     Get a list of all the child contents (recusive) of the given folder path.
 299     """
 300     args = ["list"]
 301     path = svn_path
 302     if svn_rev:
 303         args += ["-r", svn_rev]
 304         path += "@"+str(svn_rev)
 305     args += [path]
 306     paths = run_svn(args, no_fail=True)
 307     paths = paths.strip("\n").split("\n") if len(paths)>1 else []
 308     return paths
 309
 310 def path_in_list(paths, path):
 311     for p in paths:
 312         if is_child_path(path, p):
 313             return True
 314     return False
 315
 316 def add_path(paths, path):
 317     if not path_in_list(paths, path):
 318         paths.append(path)
 319
 320 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
 321                export_paths={}, is_dir = False, prefix = ""):
 322     """
 323     Given the add'd source path, replay the "svn add/copy" commands to correctly
 324     track renames across copy-from's.
 325
 326     For example, consider a sequence of events like this:
 327     1. svn copy /trunk /branches/fix1
 328     2. (Make some changes on /branches/fix1)
 329     3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2  " Rename folder
 330     4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt  " Rename file inside renamed folder
 331     5. svn co /trunk && svn merge /branches/fix1
 332     After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
 333     and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
 334     to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
 335     we'd lose the logical history that Proj2/file2.txt is really a descendant
 336     of Proj1/file1.txt.
 337
 338     'path_offset' is the offset from source_base to the file to check ancestry for,
 339       e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
 340     'source_rev' is the revision ("svn log") that we're processing from the source repo.
 341     'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
 342       directory, when being called recursively by do_svn_add_dir().
 343     'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
 344     'is_dir' is whether path_offset is a directory (rather than a file).
 345     """
 346     ui.status(prefix + ">> do_svn_add: %s  %s", source_base+"/"+path_offset+"@"+str(source_rev),
 347         "  (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
 348         level=ui.DEBUG, color='GREEN')
 349     # Check if the given path has ancestors which chain back to the current source_base
 350     found_ancestor = False
 351     ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+"  ")
 352     # ancestors[n] is the original (pre-branch-copy) trunk path.
 353     # ancestors[n-1] is the first commit on the new branch.
 354     copyfrom_path = ancestors[len(ancestors)-1]['path']     if ancestors else ""
 355     copyfrom_rev  = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
 356     if ancestors:
 357         # The copy-from path has ancestory back to source_url.
 358         ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
 359             level=ui.DEBUG, color='GREEN', bold=True)
 360         found_ancestor = True
 361         # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
 362         # return None in the case where copyfrom_rev is *before* our source_start_rev.
 363         tgt_rev = get_rev_map(copyfrom_rev, prefix+"  ")
 364         ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
 365     else:
 366         ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
 367         found_ancestor = False
 368     if found_ancestor and tgt_rev:
 369         # Check if this path_offset in the target WC already has this ancestry, in which
 370         # case there's no need to run the "svn copy" (again).
 371         path_in_svn = in_svn(path_offset, prefix=prefix+"  ")
 372         log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+"  ") else []
 373         if (not log_entry or (log_entry['revision'] != tgt_rev)):
 374             copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
 375             ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 376             ui.status(prefix + "   copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 377             ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
 378             if path_in_svn and \
 379                ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
 380                 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
 381                 # When being called recursively, if this child entry has the same ancestor as the
 382                 # the parent, then no need to try to run another "svn copy".
 383                 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
 384                     parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
 385                 pass
 386             else:
 387                 # Copy this path from the equivalent path+rev in the target repo, to create the
 388                 # equivalent history.
 389                 if parent_copyfrom_path:
 390                     # If we have a parent copy-from path, we mis-match that so display a status
 391                     # message describing the action we're mimic'ing. If path_in_svn, then this
 392                     # is logically a "replace" rather than an "add".
 393                     ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
 394                 if path_in_svn:
 395                     # If local file is already under version-control, then this is a replace.
 396                     ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
 397                     run_svn(["remove", "--force", path_offset])
 398                 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
 399                 # Export the final version of this file/folder from the source repo, to make
 400                 # sure we're up-to-date.
 401                 add_path(export_paths, path_offset)
 402         else:
 403             ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
 404     else:
 405         # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
 406         # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
 407         # Create (parent) directory if needed.
 408         # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
 409         #       split-out to a shared tag?
 410         p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
 411         if not os.path.exists(p_path):
 412             run_svn(["mkdir", p_path])
 413         if not in_svn(path_offset, prefix=prefix+"  "):
 414             if is_dir:
 415                 # Export the final verison of all files in this folder.
 416                 add_path(export_paths, path_offset)
 417             else:
 418                 # Export the final verison of this file. We *need* to do this before running
 419                 # the "svn add", even if we end-up re-exporting this file again via export_paths.
 420                 run_svn(["export", "--force", "-r", source_rev,
 421                          source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 422             # If not already under version-control, then "svn add" this file/folder.
 423             run_svn(["add", "--parents", path_offset])
 424         # TODO: Need to copy SVN properties from source repos
 425     if is_dir:
 426         # For any folders that we process, process any child contents, so that we correctly
 427         # replay copies/replaces/etc.
 428         do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, prefix+"  ")
 429
 430 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
 431                    export_paths, prefix=""):
 432     # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
 433     # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
 434     #       associated remote repo folder. (Is this a problem?)
 435     paths_local =  get_svn_dirlist(path_offset)
 436     paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
 437     ui.status(prefix + ">> do_svn_add_dir: paths_local:  %s", str(paths_local),  level=ui.DEBUG, color='GREEN')
 438     ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
 439     # Update files/folders which exist in remote but not local
 440     for path in paths_remote:
 441         path_is_dir = True if path[-1] == "/" else False
 442         working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
 443         do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
 444                    export_paths, path_is_dir, prefix+"  ")
 445     # Remove files/folders which exist in local but not remote
 446     for path in paths_local:
 447         if not path in paths_remote:
 448             ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
 449             run_svn(["remove", "--force", path_offset+"/"+path])
 450             # TODO: Does this handle deleted folders too? Wouldn't want to have a case
 451             #       where we only delete all files from folder but leave orphaned folder around.
 452
 453 def process_svn_log_entry(log_entry, options, commit_paths, prefix = ""):
 454     """
 455     Process SVN changes from the given log entry. Build an array (commit_paths)
 456     of the paths in the working-copy that were changed, i.e. the paths which
 457     we'll pass to "svn commit".
 458     """
 459     export_paths = []
 460     source_rev = log_entry['revision']
 461     ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
 462     for d in log_entry['changed_paths']:
 463         # Get the full path for this changed_path
 464         # e.g. '/branches/bug123/projectA/file1.txt'
 465         path = d['path']
 466         if not is_child_path(path, source_base):
 467             # Ignore changed files that are not part of this subdir
 468             ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s  (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
 469             continue
 470         # Note: d['kind']="" for action="M" paths which only have property changes.
 471         path_is_dir =  True if d['kind'] == 'dir'  else False
 472         path_is_file = True if d['kind'] == 'file' else False
 473         # Calculate the offset (based on source_base) for this changed_path
 474         # e.g. 'projectA/file1.txt'
 475         # (path = source_base + "/" + path_offset)
 476         path_offset = path[len(source_base):].strip("/")
 477         # Get the action for this path
 478         action = d['action']
 479         if action not in _valid_svn_actions:
 480             raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 481                 % (source_rev, action))
 482         ui.status(" %s %s%s", action, d['path'],
 483             (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 484             level=ui.VERBOSE)
 485
 486         # Try to be efficient and keep track of an explicit list of paths in the
 487         # working copy that changed. If we commit from the root of the working copy,
 488         # then SVN needs to crawl the entire working copy looking for pending changes.
 489         commit_paths.append(path_offset)
 490
 491         # Special-handling for replace's
 492         if action == 'R':
 493             # If file was "replaced" (deleted then re-added, all in same revision),
 494             # then we need to run the "svn rm" first, then change action='A'. This
 495             # lets the normal code below handle re-"svn add"'ing the files. This
 496             # should replicate the "replace".
 497             run_svn(["remove", "--force", path_offset])
 498             action = 'A'
 499
 500         # Handle all the various action-types
 501         # (Handle "add" first, for "svn copy/move" support)
 502         if action == 'A':
 503             # Determine where to export from.
 504             svn_copy = False
 505             # Handle cases where this "add" was a copy from another URL in the source repo
 506             if d['copyfrom_revision']:
 507                 copyfrom_path = d['copyfrom_path']
 508                 copyfrom_rev =  d['copyfrom_revision']
 509                 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, prefix+"  ")
 510             # Else just "svn export" the files from the source repo and "svn add" them.
 511             else:
 512                 # Create (parent) directory if needed
 513                 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
 514                 if not os.path.exists(p_path):
 515                     run_svn(["mkdir", p_path])
 516                 # Export the entire added tree.
 517                 if path_is_dir:
 518                     # For directories, defer the (recurisve) "svn export". Might have a
 519                     # situation in a branch merge where the entry in the svn-log is a
 520                     # non-copy-from'd "add" but there are child contents (that we haven't
 521                     # gotten to yet in log_entry) that are copy-from's.  When we try do
 522                     # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
 523                     # having pre-existing (svn-add'd) contents creates some trouble.
 524                     # Instead, just create the stub folders ("svn mkdir" above) and defer
 525                     # exporting the final file-state until the end.
 526                     add_path(export_paths, path_offset)
 527                 else:
 528                     # Export the final verison of this file. We *need* to do this before running
 529                     # the "svn add", even if we end-up re-exporting this file again via export_paths.
 530                     run_svn(["export", "--force", "-r", source_rev,
 531                              source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 532                 if not in_svn(path_offset, prefix=prefix+"  "):
 533                     # Need to use in_svn here to handle cases where client committed the parent
 534                     # folder and each indiv sub-folder.
 535                     run_svn(["add", "--parents", path_offset])
 536                 # TODO: Need to copy SVN properties from source repos
 537
 538         elif action == 'D':
 539             run_svn(["remove", "--force", path_offset])
 540
 541         elif action == 'M':
 542             # TODO: Is "svn merge -c" correct here? Should this just be an "svn export" plus
 543             #       proplist updating?
 544             out = run_svn(["merge", "-c", source_rev, "--non-recursive",
 545                      "--non-interactive", "--accept=theirs-full",
 546                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 547             # TODO: If d['props'] == 'modified', then run code to clean-up/purge any newly-modified props?
 548
 549         else:
 550             raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
 551                 % action)
 552
 553     # Export the final version of all add'd paths from source_url
 554     if export_paths:
 555         for path_offset in export_paths:
 556             run_svn(["export", "--force", "-r", source_rev,
 557                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 558
 559 def disp_svn_log_summary(log_entry):
 560     ui.status("")
 561     ui.status("r%s | %s | %s",
 562         log_entry['revision'],
 563         log_entry['author'],
 564         str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
 565     ui.status(log_entry['message'])
 566     ui.status("------------------------------------------------------------------------")
 567
 568 def real_main(options, args):
 569     global source_url, target_url, rev_map
 570     source_url = args.pop(0).rstrip("/")    # e.g. 'http://server/svn/source/trunk'
 571     target_url = args.pop(0).rstrip("/")    # e.g. 'file:///svn/target/trunk'
 572     ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
 573
 574     # Make sure that both the source and target URL's are valid
 575     source_info = svnclient.get_svn_info(source_url)
 576     assert is_child_path(source_url, source_info['repos_url'])
 577     target_info = svnclient.get_svn_info(target_url)
 578     assert is_child_path(target_url, target_info['repos_url'])
 579
 580     # Init global vars
 581     global source_repos_url,source_base,source_repos_uuid
 582     source_repos_url = source_info['repos_url']       # e.g. 'http://server/svn/source'
 583     source_base = source_url[len(source_repos_url):]  # e.g. '/trunk'
 584     source_repos_uuid = source_info['repos_uuid']
 585
 586     # Init start and end revision
 587     source_start_rev = svnclient.get_svn_rev(source_repos_url, options.svn_rev_start if options.svn_rev_start else 1)
 588     source_end_rev   = svnclient.get_svn_rev(source_repos_url, options.svn_rev_end   if options.svn_rev_end   else "HEAD")
 589
 590     target_end_rev = target_info['revision']   # Last revision # in the target repo
 591     wc_target = os.path.abspath('_wc_target')
 592     num_entries_proc = 0
 593     commit_count = 0
 594     source_rev = None
 595     target_rev = None
 596
 597     # Check out a working copy of target_url if needed
 598     wc_exists = os.path.exists(wc_target)
 599     if wc_exists and not options.cont_from_break:
 600         shutil.rmtree(wc_target)
 601         wc_exists = False
 602     if not wc_exists:
 603         ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
 604         svnclient.svn_checkout(target_url, wc_target)
 605     os.chdir(wc_target)
 606
 607     if not options.cont_from_break:
 608         # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
 609         # Get the first log entry at/after source_start_rev, which is where
 610         # we'll do the initial import from.
 611         it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False)
 612         for source_start_log in it_log_start:
 613             break
 614         if not source_start_log:
 615             raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
 616                 (source_start_rev, source_end_rev, source_url))
 617
 618         # This is the revision we will start from for source_url
 619         source_start_rev = source_rev = int(source_start_log['revision'])
 620         ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
 621
 622         # For the initial commit to the target URL, export all the contents from
 623         # the source URL at the start-revision.
 624         paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
 625         if len(paths)>1:
 626             disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
 627             ui.status("(Initial import)", level=ui.VERBOSE)
 628             paths = paths.strip("\n").split("\n")
 629             for path_raw in paths:
 630                 # For each top-level file/folder...
 631                 if not path_raw:
 632                     continue
 633                 # Directories have a trailing slash in the "svn list" output
 634                 path_is_dir = True if path_raw[-1] == "/" else False
 635                 path = path_raw.rstrip('/') if path_is_dir else path_raw
 636                 if path_is_dir and not os.path.exists(path):
 637                     os.makedirs(path)
 638                 ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path, level=ui.VERBOSE)
 639                 run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path+"@"+str(source_rev), path])
 640                 run_svn(["add", path])
 641             num_entries_proc += 1
 642             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 643             target_rev = commit_from_svn_log_entry(source_start_log, options, target_revprops=target_revprops)
 644             if target_rev:
 645                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 646                 set_rev_map(source_rev, target_rev)
 647                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 648                 run_svn(["update"])
 649                 commit_count += 1
 650     else:
 651         # Re-build the rev_map based on any already-replayed history in target_url
 652         build_rev_map(target_url, target_end_rev, source_info)
 653         if not rev_map:
 654             raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url)
 655         source_start_rev = int(max(rev_map, key=rev_map.get))
 656         assert source_start_rev
 657         ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
 658
 659     svn_vers_t = svnclient.get_svn_client_version()
 660     svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
 661
 662     # Load SVN log starting from source_start_rev + 1
 663     it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else []
 664     source_rev = None
 665
 666     try:
 667         for log_entry in it_log_entries:
 668             if options.entries_proc_limit:
 669                 if num_entries_proc >= options.entries_proc_limit:
 670                     break
 671             # Replay this revision from source_url into target_url
 672             disp_svn_log_summary(log_entry)
 673             source_rev = log_entry['revision']
 674             # Process all the changed-paths in this log entry
 675             commit_paths = []
 676             process_svn_log_entry(log_entry, options, commit_paths)
 677             num_entries_proc += 1
 678             # Commit any changes made to _wc_target
 679             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 680             target_rev = commit_from_svn_log_entry(log_entry, options, commit_paths, target_revprops=target_revprops)
 681             if target_rev:
 682                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 683                 source_rev = log_entry['revision']
 684                 set_rev_map(source_rev, target_rev)
 685                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 686                 run_svn(["update"])
 687                 commit_count += 1
 688                 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
 689                 if svn_vers >= 1.7 and (commit_count % 100 == 0):
 690                     run_svn(["cleanup"])
 691         if not source_rev:
 692             # If there were no new source_url revisions to process, init source_rev
 693             # for the "finally" message below.
 694             source_rev = source_end_rev
 695
 696     except KeyboardInterrupt:
 697         print "\nStopped by user."
 698         print "\nCleaning-up..."
 699         run_svn(["cleanup"])
 700         full_svn_revert()
 701     except:
 702         print "\nCommand failed with following error:\n"
 703         traceback.print_exc()
 704         print "\nCleaning-up..."
 705         run_svn(["cleanup"])
 706         print run_svn(["status"])
 707         full_svn_revert()
 708     finally:
 709         print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
 710
 711 def main():
 712     # Defined as entry point. Must be callable without arguments.
 713     usage = "Usage: %prog [OPTIONS] source_url target_url"
 714     description = """\
 715   Replicate (replay) history from one SVN repository to another. Maintain
 716   logical ancestry wherever possible, so that 'svn log' on the replayed
 717   repo will correctly follow file/folder renames.
 718
 719   == Examples ==
 720   Create a copy of only /trunk from source repo, starting at r5000
 721   $ svnadmin create /svn/target
 722   $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
 723   $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
 724     1. The target_url will be checked-out to ./_wc_target
 725     2. The first commit to http://server/source/trunk at/after r5000 will be
 726        exported & added into _wc_target
 727     3. All revisions affecting http://server/source/trunk (starting at r5000)
 728        will be replayed to _wc_target. Any add/copy/move/replaces that are
 729        copy-from'd some path outside of /trunk (e.g. files renamed on a
 730        /branch and branch was merged into /trunk) will correctly maintain
 731        logical ancestry where possible.
 732
 733   Use continue-mode (-c) to pick-up where the last run left-off
 734   $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
 735     1. The target_url will be checked-out to ./_wc_target, if not already
 736        checked-out
 737     2. All new revisions affecting http://server/source/trunk starting from
 738        the last replayed revision to file:///svn/target/trunk (based on the
 739        svn2svn:* revprops) will be replayed to _wc_target, maintaining all
 740        logical ancestry where possible."""
 741     parser = optparse.OptionParser(usage, description=description,
 742                 formatter=HelpFormatter(), version="%prog "+str(full_version))
 743     #parser.remove_option("--help")
 744     #parser.add_option("-h", "--help", dest="show_help", action="store_true",
 745     #    help="show this help message and exit")
 746     parser.add_option("-r", "--revision", type="string", dest="svn_rev", metavar="ARG",
 747                       help="revision range to replay from source_url\n" + \
 748                            "A revision argument can be one of:\n" + \
 749                            "   START        start rev # (end will be 'HEAD')\n" + \
 750                            "   START:END    start and ending rev #'s\n" + \
 751                            "(Any revision # formats which SVN understands\n" + \
 752                            " are supported, e.g. 'HEAD', '{2010-01-31}', etc.)")
 753     parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", default=False,
 754                       help="maintain original 'Author' info from source repo")
 755     parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
 756                       help="continue from previous break")
 757     parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
 758                       help="maximum number of log entries to process")
 759     parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 760                       help="try processing next log entry but don't commit changes to "
 761                            "target working-copy (forces --limit=1)")
 762     parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
 763                       help="enable additional output (use -vv or -vvv for more)")
 764     parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
 765                       help="enable debugging output (same as -vvv)")
 766     options, args = parser.parse_args()
 767     if len(args) != 2:
 768         parser.error("incorrect number of arguments")
 769     if options.verbosity < 10:
 770         # Expand multiple "-v" arguments to a real ui._level value
 771         options.verbosity *= 10
 772     if options.dry_run:
 773         # When in dry-run mode, only try to process the next log_entry
 774         options.entries_proc_limit = 1
 775     options.svn_rev_start = None
 776     options.svn_rev_end   = None
 777     if options.svn_rev:
 778         rev = options.svn_rev.split(":")
 779         options.svn_rev_start = rev[0] if len(rev)>0 else None
 780         options.svn_rev_end   = rev[1] if len(rev)>1 else None
 781     ui.update_config(options)
 782     return real_main(options, args)
 783
 784
 785 if __name__ == "__main__":
 786     sys.exit(main() or 0)