svn2svn/run/svn2svn.py

   1 """
   2 Replicate (replay) changesets from one SVN repository to another.
   3 """
   4
   5 from .. import base_version, full_version
   6 from .. import ui
   7 from .. import svnclient
   8 from ..shell import run_svn
   9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
  10 from parse import HelpFormatter
  11
  12 import sys
  13 import os
  14 import time
  15 import traceback
  16 import shutil
  17 import operator
  18 import optparse
  19 from datetime import datetime
  20
  21 _valid_svn_actions = "MARD"   # The list of known SVN action abbr's, from "svn log"
  22
  23 # Module-level variables/parameters
  24 source_url = ""          # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
  25 source_repos_url = ""    # URL to root of source SVN repo,        e.g. 'http://server/svn/source'
  26 source_base = ""         # Relative path of source_url in source SVN repo, e.g. '/trunk'
  27 source_repos_uuid = ""   # UUID of source SVN repo
  28 target_url =""           # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
  29 rev_map = {}             # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
  30
  31 def commit_from_svn_log_entry(log_entry, options, commit_paths=None, target_revprops=None):
  32     """
  33     Given an SVN log entry and an optional list of changed paths, do an svn commit.
  34     """
  35     # TODO: Run optional external shell hook here, for doing pre-commit filtering
  36     # Display the _wc_target "svn status" info if running in -vv (or higher) mode
  37     if ui.get_level() >= ui.EXTRA:
  38         ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
  39         ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
  40     # This will use the local timezone for displaying commit times
  41     timestamp = int(log_entry['date'])
  42     svn_date = str(datetime.fromtimestamp(timestamp))
  43     # Uncomment this one one if you prefer UTC commit times
  44     #svn_date = "%d 0" % timestamp
  45     args = ["commit", "--force-log"]
  46     if options.keep_author:
  47         args += ["-m", log_entry['message'] + "\nDate: " + svn_date, "--username", log_entry['author']]
  48     else:
  49         args += ["-m", log_entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + log_entry['author']]
  50     revprops = {}
  51     if log_entry['revprops']:
  52         # Carry forward any revprop's from the source revision
  53         for v in log_entry['revprops']:
  54             revprops[v['name']] = v['value']
  55     if target_revprops:
  56         # Add any extra revprop's we want to set for the target repo commits
  57         for v in target_revprops:
  58             revprops[v['name']] = v['value']
  59     if revprops:
  60         for key in revprops:
  61             args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
  62     if commit_paths:
  63         if len(commit_paths)<100:
  64             # If we don't have an excessive amount of individual changed paths, pass
  65             # those to the "svn commit" command. Else, pass nothing so we commit at
  66             # the root of the working-copy.
  67             args += list(commit_paths)
  68     rev = None
  69     if not options.dry_run:
  70         # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
  71         output = run_svn(args)
  72         if output:
  73             output_lines = output.strip("\n").split("\n")
  74             rev = ""
  75             for line in output_lines:
  76                 if line[0:19] == 'Committed revision ':
  77                     rev = line[19:].rstrip('.')
  78                     break
  79             if rev:
  80                 ui.status("Committed revision %s.", rev)
  81     return rev
  82
  83 def full_svn_revert():
  84     """
  85     Do an "svn revert" and proactively remove any extra files in the working copy.
  86     """
  87     run_svn(["revert", "--recursive", "."])
  88     output = run_svn(["status"])
  89     if output:
  90         output_lines = output.strip("\n").split("\n")
  91         for line in output_lines:
  92             if line[0] == "?":
  93                 path = line[4:].strip(" ")
  94                 if os.path.isfile(path):
  95                     os.remove(path)
  96                 if os.path.isdir(path):
  97                     shutil.rmtree(path)
  98
  99 def gen_tracking_revprops(source_rev):
 100     """
 101     Build an array of svn2svn-specific source-tracking revprops.
 102     """
 103     revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
 104                 {'name':'svn2svn:source_url',  'value':source_url},
 105                 {'name':'svn2svn:source_rev',  'value':source_rev}]
 106     return revprops
 107
 108 def in_svn(p, require_in_repo=False, prefix=""):
 109     """
 110     Check if a given file/folder is being tracked by Subversion.
 111     Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
 112     With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
 113     Use "svn status" to check the status of the file/folder.
 114     """
 115     entries = svnclient.get_svn_status(p, no_recursive=True)
 116     if not entries:
 117         return False
 118     d = entries[0]
 119     if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
 120         # If caller requires this path to be in the SVN repo, prevent returning True
 121         # for paths that are only locally-added.
 122         ret = False
 123     else:
 124         # Don't consider files tracked as deleted in the WC as under source-control.
 125         # Consider files which are locally added/copied as under source-control.
 126         ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
 127     ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
 128     return ret
 129
 130 def is_child_path(path, p_path):
 131     return True if (path == p_path) or (path.startswith(p_path+"/")) else False
 132
 133 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
 134     """
 135     Given a source path, walk the SVN history backwards to inspect the ancestory of
 136     that path, seeing if it traces back to base_path.  Build an array of copyfrom_path
 137     and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
 138     which base_path is a substring match of (e.g. we crawled back to the initial branch-
 139     copy from trunk), then return the collection of ancestor paths.  Otherwise,
 140     copyfrom_path has no ancestory compared to base_path.
 141
 142     This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
 143     file/folder was renamed in a branch and then that branch was merged back to trunk.
 144
 145     'svn_repos_url' is the full URL to the root of the SVN repository,
 146       e.g. 'file:///path/to/repo'
 147     'base_path' is the path in the SVN repo to the target path we're trying to
 148       trace ancestry back to, e.g. 'trunk'.
 149     'source_path' is the path in the SVN repo to the source path to start checking
 150       ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
 151       (full_path = svn_repos_url+base_path+"/"+path_offset)
 152     'source_rev' is the revision to start walking the history of source_path backwards from.
 153     """
 154     ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s  base_path: %s",
 155         svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
 156     done = False
 157     working_path = base_path+"/"+source_path
 158     working_rev  = source_rev
 159     first_iter_done = False
 160     ancestors_temp = []
 161     while not done:
 162         # Get the first "svn log" entry for this path (relative to @rev)
 163         ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
 164         log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path, 1, working_rev, True)
 165         if not log_entry:
 166             ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
 167             done = True
 168             break
 169         # If we found a copy-from case which matches our base_path, we're done.
 170         # ...but only if we've at least tried to search for the first copy-from path.
 171         if first_iter_done and is_child_path(working_path, base_path):
 172             ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(working_path, base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
 173             done = True
 174             break
 175         first_iter_done = True
 176         # Search for any actions on our target path (or parent paths).
 177         changed_paths_temp = []
 178         for d in log_entry['changed_paths']:
 179             path = d['path']
 180             if path in working_path:
 181                 changed_paths_temp.append({'path': path, 'data': d})
 182         if not changed_paths_temp:
 183             # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
 184             ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
 185             done = True
 186             continue
 187         # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
 188         changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
 189         # Find the action for our working_path in this revision. Use a loop to check in reverse order,
 190         # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
 191         for v in changed_paths:
 192             d = v['data']
 193             path = d['path']
 194             # Check action-type for this file
 195             action = d['action']
 196             if action not in _valid_svn_actions:
 197                 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 198                     % (log_entry['revision'], action))
 199             ui.status(prefix + "> %s %s%s", action, path,
 200                 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 201                 level=ui.DEBUG, color='YELLOW')
 202             if action == 'D':
 203                 # If file/folder was deleted, it has no ancestor
 204                 ancestors_temp = []
 205                 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
 206                 done = True
 207                 break
 208             if action in 'RA':
 209                 # If file/folder was added/replaced but not a copy, it has no ancestor
 210                 if not d['copyfrom_path']:
 211                     ancestors_temp = []
 212                     ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
 213                         "Added" if action == "A" else "Replaced",
 214                         level=ui.DEBUG, color='YELLOW')
 215                     done = True
 216                     break
 217                 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
 218                 # and keep checking for ancestors
 219                 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
 220                     action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
 221                     level=ui.DEBUG, color='YELLOW')
 222                 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
 223                                        'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
 224                 working_path = working_path.replace(d['path'], d['copyfrom_path'])
 225                 working_rev =  d['copyfrom_revision']
 226                 # Follow the copy and keep on searching
 227                 break
 228     ancestors = []
 229     if ancestors_temp:
 230         ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
 231         working_path = base_path+"/"+source_path
 232         for idx in range(len(ancestors_temp)):
 233             d = ancestors_temp[idx]
 234             working_path = working_path.replace(d['path'], d['copyfrom_path'])
 235             working_rev =  d['copyfrom_rev']
 236             ancestors.append({'path': working_path, 'revision': working_rev})
 237         if ui.get_level() >= ui.DEBUG:
 238             max_len = 0
 239             for idx in range(len(ancestors)):
 240                 d = ancestors[idx]
 241                 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
 242             ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
 243             for idx in range(len(ancestors)-1):
 244                 d = ancestors[idx]
 245                 d_next = ancestors[idx+1]
 246                 ui.status(prefix + " [%s] %s <-- %s", idx,
 247                     str(d['path']+"@"+str(d['revision'])).ljust(max_len),
 248                     str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
 249                     level=ui.DEBUG, color='YELLOW')
 250     else:
 251         ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
 252             svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
 253     return ancestors
 254
 255 def get_rev_map(source_rev, prefix):
 256     """
 257     Find the equivalent rev # in the target repo for the given rev # from the source repo.
 258     """
 259     ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
 260     # Find the highest entry less-than-or-equal-to source_rev
 261     for rev in range(int(source_rev), 0, -1):
 262         ui.status(prefix + ">> get_rev_map: rev=%s  in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
 263         if rev in rev_map:
 264             return int(rev_map[rev])
 265     # Else, we fell off the bottom of the rev_map. Ruh-roh...
 266     return None
 267
 268 def set_rev_map(source_rev, target_rev):
 269     ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
 270     global rev_map
 271     rev_map[int(source_rev)]=int(target_rev)
 272
 273 def build_rev_map(target_url, target_end_rev, source_info):
 274     """
 275     Check for any already-replayed history from source_url (source_info) and
 276     build the mapping-table of source_rev -> target_rev.
 277     """
 278     global rev_map
 279     rev_map = {}
 280     ui.status("Rebuilding rev_map...", level=ui.VERBOSE)
 281     proc_count = 0
 282     it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
 283     for log_entry in it_log_entries:
 284         if log_entry['revprops']:
 285             revprops = {}
 286             for v in log_entry['revprops']:
 287                 if v['name'].startswith('svn2svn:'):
 288                     revprops[v['name']] = v['value']
 289             if revprops and \
 290                revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
 291                revprops['svn2svn:source_url'] == source_info['url']:
 292                 source_rev = revprops['svn2svn:source_rev']
 293                 target_rev = log_entry['revision']
 294                 set_rev_map(source_rev, target_rev)
 295
 296 def get_svn_dirlist(svn_path, svn_rev = ""):
 297     """
 298     Get a list of all the child contents (recusive) of the given folder path.
 299     """
 300     args = ["list"]
 301     path = svn_path
 302     if svn_rev:
 303         args += ["-r", svn_rev]
 304         path += "@"+str(svn_rev)
 305     args += [path]
 306     paths = run_svn(args, no_fail=True)
 307     paths = paths.strip("\n").split("\n") if len(paths)>1 else []
 308     return paths
 309
 310 def path_in_list(paths, path):
 311     for p in paths:
 312         if is_child_path(path, p):
 313             return True
 314     return False
 315
 316 def add_path(paths, path):
 317     if not path_in_list(paths, path):
 318         paths.append(path)
 319
 320 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
 321                export_paths={}, is_dir = False, prefix = ""):
 322     """
 323     Given the add'd source path, replay the "svn add/copy" commands to correctly
 324     track renames across copy-from's.
 325
 326     For example, consider a sequence of events like this:
 327     1. svn copy /trunk /branches/fix1
 328     2. (Make some changes on /branches/fix1)
 329     3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2  " Rename folder
 330     4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt  " Rename file inside renamed folder
 331     5. svn co /trunk && svn merge /branches/fix1
 332     After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
 333     and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
 334     to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
 335     we'd lose the logical history that Proj2/file2.txt is really a descendant
 336     of Proj1/file1.txt.
 337
 338     'path_offset' is the offset from source_base to the file to check ancestry for,
 339       e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
 340     'source_rev' is the revision ("svn log") that we're processing from the source repo.
 341     'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
 342       directory, when being called recursively by do_svn_add_dir().
 343     'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
 344     'is_dir' is whether path_offset is a directory (rather than a file).
 345     """
 346     ui.status(prefix + ">> do_svn_add: %s  %s", source_base+"/"+path_offset+"@"+str(source_rev),
 347         "  (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
 348         level=ui.DEBUG, color='GREEN')
 349     # Check if the given path has ancestors which chain back to the current source_base
 350     found_ancestor = False
 351     ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+"  ")
 352     # ancestors[n] is the original (pre-branch-copy) trunk path.
 353     # ancestors[n-1] is the first commit on the new branch.
 354     copyfrom_path = ancestors[len(ancestors)-1]['path']     if ancestors else ""
 355     copyfrom_rev  = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
 356     if ancestors:
 357         # The copy-from path has ancestory back to source_url.
 358         ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
 359             level=ui.DEBUG, color='GREEN', bold=True)
 360         found_ancestor = True
 361         # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
 362         # return None in the case where copyfrom_rev is *before* our source_start_rev.
 363         tgt_rev = get_rev_map(copyfrom_rev, prefix+"  ")
 364         ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
 365     else:
 366         ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
 367         found_ancestor = False
 368     if found_ancestor and tgt_rev:
 369         # Check if this path_offset in the target WC already has this ancestry, in which
 370         # case there's no need to run the "svn copy" (again).
 371         path_in_svn = in_svn(path_offset, prefix=prefix+"  ")
 372         log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+"  ") else []
 373         if (not log_entry or (log_entry['revision'] != tgt_rev)):
 374             copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
 375             ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 376             ui.status(prefix + "   copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 377             ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
 378             if path_in_svn and \
 379                ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
 380                 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
 381                 # When being called recursively, if this child entry has the same ancestor as the
 382                 # the parent, then no need to try to run another "svn copy".
 383                 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
 384                     parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
 385                 pass
 386             else:
 387                 # Copy this path from the equivalent path+rev in the target repo, to create the
 388                 # equivalent history.
 389                 if parent_copyfrom_path:
 390                     # If we have a parent copy-from path, we mis-match that so display a status
 391                     # message describing the action we're mimic'ing. If path_in_svn, then this
 392                     # is logically a "replace" rather than an "add".
 393                     ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
 394                 if path_in_svn:
 395                     # If local file is already under version-control, then this is a replace.
 396                     ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
 397                     run_svn(["remove", "--force", path_offset])
 398                 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
 399                 # Export the final version of this file/folder from the source repo, to make
 400                 # sure we're up-to-date.
 401                 add_path(export_paths, path_offset)
 402         else:
 403             ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
 404     else:
 405         # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
 406         # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
 407         # Create (parent) directory if needed.
 408         # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
 409         #       split-out to a shared tag?
 410         p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
 411         if not os.path.exists(p_path):
 412             run_svn(["mkdir", p_path])
 413         if not in_svn(path_offset, prefix=prefix+"  "):
 414             if is_dir:
 415                 # Export the final verison of all files in this folder.
 416                 add_path(export_paths, path_offset)
 417             else:
 418                 # Export the final verison of this file. We *need* to do this before running
 419                 # the "svn add", even if we end-up re-exporting this file again via export_paths.
 420                 run_svn(["export", "--force", "-r", source_rev,
 421                          source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 422             # If not already under version-control, then "svn add" this file/folder.
 423             run_svn(["add", "--parents", path_offset])
 424         # TODO: Need to copy SVN properties from source repos
 425     if is_dir:
 426         # For any folders that we process, process any child contents, so that we correctly
 427         # replay copies/replaces/etc.
 428         do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, prefix+"  ")
 429
 430 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
 431                    export_paths, prefix=""):
 432     # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
 433     # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
 434     #       associated remote repo folder. (Is this a problem?)
 435     paths_local =  get_svn_dirlist(path_offset)
 436     paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
 437     ui.status(prefix + ">> do_svn_add_dir: paths_local:  %s", str(paths_local),  level=ui.DEBUG, color='GREEN')
 438     ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
 439     # Update files/folders which exist in remote but not local
 440     for path in paths_remote:
 441         path_is_dir = True if path[-1] == "/" else False
 442         working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
 443         do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
 444                    export_paths, path_is_dir, prefix+"  ")
 445     # Remove files/folders which exist in local but not remote
 446     for path in paths_local:
 447         if not path in paths_remote:
 448             ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
 449             run_svn(["remove", "--force", path_offset+"/"+path])
 450             # TODO: Does this handle deleted folders too? Wouldn't want to have a case
 451             #       where we only delete all files from folder but leave orphaned folder around.
 452
 453 def process_svn_log_entry(log_entry, options, commit_paths, prefix = ""):
 454     """
 455     Process SVN changes from the given log entry. Build an array (commit_paths)
 456     of the paths in the working-copy that were changed, i.e. the paths which
 457     we'll pass to "svn commit".
 458     """
 459     export_paths = []
 460     source_rev = log_entry['revision']
 461     ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
 462     for d in log_entry['changed_paths']:
 463         # Get the full path for this changed_path
 464         # e.g. '/branches/bug123/projectA/file1.txt'
 465         path = d['path']
 466         if not is_child_path(path, source_base):
 467             # Ignore changed files that are not part of this subdir
 468             ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s  (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
 469             continue
 470         # Note: d['kind']="" for action="M" paths which only have property changes.
 471         if d['kind'] == "":
 472             d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
 473         assert (d['kind'] == 'file') or (d['kind'] == 'dir')
 474         path_is_dir =  True if d['kind'] == 'dir'  else False
 475         path_is_file = True if d['kind'] == 'file' else False
 476         # Calculate the offset (based on source_base) for this changed_path
 477         # e.g. 'projectA/file1.txt'
 478         # (path = source_base + "/" + path_offset)
 479         path_offset = path[len(source_base):].strip("/")
 480         # Get the action for this path
 481         action = d['action']
 482         if action not in _valid_svn_actions:
 483             raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 484                 % (source_rev, action))
 485         ui.status(" %s %s%s", action, d['path'],
 486             (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 487             level=ui.VERBOSE)
 488
 489         # Try to be efficient and keep track of an explicit list of paths in the
 490         # working copy that changed. If we commit from the root of the working copy,
 491         # then SVN needs to crawl the entire working copy looking for pending changes.
 492         commit_paths.append(path_offset)
 493
 494         # Special-handling for replace's
 495         if action == 'R':
 496             # If file was "replaced" (deleted then re-added, all in same revision),
 497             # then we need to run the "svn rm" first, then change action='A'. This
 498             # lets the normal code below handle re-"svn add"'ing the files. This
 499             # should replicate the "replace".
 500             run_svn(["remove", "--force", path_offset])
 501             action = 'A'
 502
 503         # Handle all the various action-types
 504         # (Handle "add" first, for "svn copy/move" support)
 505         if action == 'A':
 506             # Determine where to export from.
 507             svn_copy = False
 508             # Handle cases where this "add" was a copy from another URL in the source repo
 509             if d['copyfrom_revision']:
 510                 copyfrom_path = d['copyfrom_path']
 511                 copyfrom_rev =  d['copyfrom_revision']
 512                 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, prefix+"  ")
 513             # Else just "svn export" the files from the source repo and "svn add" them.
 514             else:
 515                 # Create (parent) directory if needed
 516                 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
 517                 if not os.path.exists(p_path):
 518                     run_svn(["mkdir", p_path])
 519                 # Export the entire added tree.
 520                 if path_is_dir:
 521                     # For directories, defer the (recurisve) "svn export". Might have a
 522                     # situation in a branch merge where the entry in the svn-log is a
 523                     # non-copy-from'd "add" but there are child contents (that we haven't
 524                     # gotten to yet in log_entry) that are copy-from's.  When we try do
 525                     # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
 526                     # having pre-existing (svn-add'd) contents creates some trouble.
 527                     # Instead, just create the stub folders ("svn mkdir" above) and defer
 528                     # exporting the final file-state until the end.
 529                     add_path(export_paths, path_offset)
 530                 else:
 531                     # Export the final verison of this file. We *need* to do this before running
 532                     # the "svn add", even if we end-up re-exporting this file again via export_paths.
 533                     run_svn(["export", "--force", "-r", source_rev,
 534                              source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 535                 if not in_svn(path_offset, prefix=prefix+"  "):
 536                     # Need to use in_svn here to handle cases where client committed the parent
 537                     # folder and each indiv sub-folder.
 538                     run_svn(["add", "--parents", path_offset])
 539                 # TODO: Need to copy SVN properties from source repos
 540
 541         elif action == 'D':
 542             run_svn(["remove", "--force", path_offset])
 543
 544         elif action == 'M':
 545             # TODO: Is "svn merge -c" correct here? Should this just be an "svn export" plus
 546             #       proplist updating?
 547             out = run_svn(["merge", "-c", source_rev, "--non-recursive",
 548                      "--non-interactive", "--accept=theirs-full",
 549                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 550             # TODO: If d['props'] == 'modified', then run code to clean-up/purge any newly-modified props?
 551
 552         else:
 553             raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
 554                 % action)
 555
 556     # Export the final version of all add'd paths from source_url
 557     if export_paths:
 558         for path_offset in export_paths:
 559             run_svn(["export", "--force", "-r", source_rev,
 560                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 561
 562 def disp_svn_log_summary(log_entry):
 563     ui.status("")
 564     ui.status("r%s | %s | %s",
 565         log_entry['revision'],
 566         log_entry['author'],
 567         str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
 568     ui.status(log_entry['message'])
 569     ui.status("------------------------------------------------------------------------")
 570
 571 def real_main(options, args):
 572     global source_url, target_url, rev_map
 573     source_url = args.pop(0).rstrip("/")    # e.g. 'http://server/svn/source/trunk'
 574     target_url = args.pop(0).rstrip("/")    # e.g. 'file:///svn/target/trunk'
 575     ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
 576
 577     # Make sure that both the source and target URL's are valid
 578     source_info = svnclient.get_svn_info(source_url)
 579     assert is_child_path(source_url, source_info['repos_url'])
 580     target_info = svnclient.get_svn_info(target_url)
 581     assert is_child_path(target_url, target_info['repos_url'])
 582
 583     # Init global vars
 584     global source_repos_url,source_base,source_repos_uuid
 585     source_repos_url = source_info['repos_url']       # e.g. 'http://server/svn/source'
 586     source_base = source_url[len(source_repos_url):]  # e.g. '/trunk'
 587     source_repos_uuid = source_info['repos_uuid']
 588
 589     # Init start and end revision
 590     source_start_rev = svnclient.get_svn_rev(source_repos_url, options.svn_rev_start if options.svn_rev_start else 1)
 591     source_end_rev   = svnclient.get_svn_rev(source_repos_url, options.svn_rev_end   if options.svn_rev_end   else "HEAD")
 592
 593     target_end_rev = target_info['revision']   # Last revision # in the target repo
 594     wc_target = os.path.abspath('_wc_target')
 595     num_entries_proc = 0
 596     commit_count = 0
 597     source_rev = None
 598     target_rev = None
 599
 600     # Check out a working copy of target_url if needed
 601     wc_exists = os.path.exists(wc_target)
 602     if wc_exists and not options.cont_from_break:
 603         shutil.rmtree(wc_target)
 604         wc_exists = False
 605     if not wc_exists:
 606         ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
 607         svnclient.svn_checkout(target_url, wc_target)
 608     os.chdir(wc_target)
 609
 610     if not options.cont_from_break:
 611         # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
 612         # Get the first log entry at/after source_start_rev, which is where
 613         # we'll do the initial import from.
 614         it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False)
 615         for source_start_log in it_log_start:
 616             break
 617         if not source_start_log:
 618             raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
 619                 (source_start_rev, source_end_rev, source_url))
 620
 621         # This is the revision we will start from for source_url
 622         source_start_rev = source_rev = int(source_start_log['revision'])
 623         ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
 624
 625         # For the initial commit to the target URL, export all the contents from
 626         # the source URL at the start-revision.
 627         paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
 628         if len(paths)>1:
 629             disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
 630             ui.status("(Initial import)", level=ui.VERBOSE)
 631             paths = paths.strip("\n").split("\n")
 632             for path_raw in paths:
 633                 # For each top-level file/folder...
 634                 if not path_raw:
 635                     continue
 636                 # Directories have a trailing slash in the "svn list" output
 637                 path_is_dir = True if path_raw[-1] == "/" else False
 638                 path = path_raw.rstrip('/') if path_is_dir else path_raw
 639                 if path_is_dir and not os.path.exists(path):
 640                     os.makedirs(path)
 641                 ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path, level=ui.VERBOSE)
 642                 run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path+"@"+str(source_rev), path])
 643                 run_svn(["add", path])
 644             num_entries_proc += 1
 645             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 646             target_rev = commit_from_svn_log_entry(source_start_log, options, target_revprops=target_revprops)
 647             if target_rev:
 648                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 649                 set_rev_map(source_rev, target_rev)
 650                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 651                 run_svn(["update"])
 652                 commit_count += 1
 653     else:
 654         # Re-build the rev_map based on any already-replayed history in target_url
 655         build_rev_map(target_url, target_end_rev, source_info)
 656         if not rev_map:
 657             raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url)
 658         source_start_rev = int(max(rev_map, key=rev_map.get))
 659         assert source_start_rev
 660         ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
 661
 662     svn_vers_t = svnclient.get_svn_client_version()
 663     svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
 664
 665     # Load SVN log starting from source_start_rev + 1
 666     it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else []
 667     source_rev = None
 668
 669     try:
 670         for log_entry in it_log_entries:
 671             if options.entries_proc_limit:
 672                 if num_entries_proc >= options.entries_proc_limit:
 673                     break
 674             # Replay this revision from source_url into target_url
 675             disp_svn_log_summary(log_entry)
 676             source_rev = log_entry['revision']
 677             # Process all the changed-paths in this log entry
 678             commit_paths = []
 679             process_svn_log_entry(log_entry, options, commit_paths)
 680             num_entries_proc += 1
 681             # Commit any changes made to _wc_target
 682             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 683             target_rev = commit_from_svn_log_entry(log_entry, options, commit_paths, target_revprops=target_revprops)
 684             if target_rev:
 685                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 686                 source_rev = log_entry['revision']
 687                 set_rev_map(source_rev, target_rev)
 688                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 689                 run_svn(["update"])
 690                 commit_count += 1
 691                 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
 692                 if svn_vers >= 1.7 and (commit_count % 100 == 0):
 693                     run_svn(["cleanup"])
 694         if not source_rev:
 695             # If there were no new source_url revisions to process, init source_rev
 696             # for the "finally" message below.
 697             source_rev = source_end_rev
 698
 699     except KeyboardInterrupt:
 700         print "\nStopped by user."
 701         print "\nCleaning-up..."
 702         run_svn(["cleanup"])
 703         full_svn_revert()
 704     except:
 705         print "\nCommand failed with following error:\n"
 706         traceback.print_exc()
 707         print "\nCleaning-up..."
 708         run_svn(["cleanup"])
 709         print run_svn(["status"])
 710         full_svn_revert()
 711     finally:
 712         print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
 713
 714 def main():
 715     # Defined as entry point. Must be callable without arguments.
 716     usage = "Usage: %prog [OPTIONS] source_url target_url"
 717     description = """\
 718   Replicate (replay) history from one SVN repository to another. Maintain
 719   logical ancestry wherever possible, so that 'svn log' on the replayed
 720   repo will correctly follow file/folder renames.
 721
 722   == Examples ==
 723   Create a copy of only /trunk from source repo, starting at r5000
 724   $ svnadmin create /svn/target
 725   $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
 726   $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
 727     1. The target_url will be checked-out to ./_wc_target
 728     2. The first commit to http://server/source/trunk at/after r5000 will be
 729        exported & added into _wc_target
 730     3. All revisions affecting http://server/source/trunk (starting at r5000)
 731        will be replayed to _wc_target. Any add/copy/move/replaces that are
 732        copy-from'd some path outside of /trunk (e.g. files renamed on a
 733        /branch and branch was merged into /trunk) will correctly maintain
 734        logical ancestry where possible.
 735
 736   Use continue-mode (-c) to pick-up where the last run left-off
 737   $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
 738     1. The target_url will be checked-out to ./_wc_target, if not already
 739        checked-out
 740     2. All new revisions affecting http://server/source/trunk starting from
 741        the last replayed revision to file:///svn/target/trunk (based on the
 742        svn2svn:* revprops) will be replayed to _wc_target, maintaining all
 743        logical ancestry where possible."""
 744     parser = optparse.OptionParser(usage, description=description,
 745                 formatter=HelpFormatter(), version="%prog "+str(full_version))
 746     #parser.remove_option("--help")
 747     #parser.add_option("-h", "--help", dest="show_help", action="store_true",
 748     #    help="show this help message and exit")
 749     parser.add_option("-r", "--revision", type="string", dest="svn_rev", metavar="ARG",
 750                       help="revision range to replay from source_url\n" + \
 751                            "A revision argument can be one of:\n" + \
 752                            "   START        start rev # (end will be 'HEAD')\n" + \
 753                            "   START:END    start and ending rev #'s\n" + \
 754                            "(Any revision # formats which SVN understands\n" + \
 755                            " are supported, e.g. 'HEAD', '{2010-01-31}', etc.)")
 756     parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", default=False,
 757                       help="maintain original 'Author' info from source repo")
 758     parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
 759                       help="continue from previous break")
 760     parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
 761                       help="maximum number of log entries to process")
 762     parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 763                       help="try processing next log entry but don't commit changes to "
 764                            "target working-copy (forces --limit=1)")
 765     parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
 766                       help="enable additional output (use -vv or -vvv for more)")
 767     parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
 768                       help="enable debugging output (same as -vvv)")
 769     options, args = parser.parse_args()
 770     if len(args) != 2:
 771         parser.error("incorrect number of arguments")
 772     if options.verbosity < 10:
 773         # Expand multiple "-v" arguments to a real ui._level value
 774         options.verbosity *= 10
 775     if options.dry_run:
 776         # When in dry-run mode, only try to process the next log_entry
 777         options.entries_proc_limit = 1
 778     options.svn_rev_start = None
 779     options.svn_rev_end   = None
 780     if options.svn_rev:
 781         rev = options.svn_rev.split(":")
 782         options.svn_rev_start = rev[0] if len(rev)>0 else None
 783         options.svn_rev_end   = rev[1] if len(rev)>1 else None
 784     ui.update_config(options)
 785     return real_main(options, args)
 786
 787
 788 if __name__ == "__main__":
 789     sys.exit(main() or 0)