svn2svn/run/svn2svn.py

   1 """
   2 Replicate (replay) changesets from one SVN repository to another.
   3 """
   4
   5 from .. import base_version, full_version
   6 from .. import ui
   7 from .. import svnclient
   8 from ..shell import run_svn
   9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
  10 from parse import HelpFormatter
  11
  12 import sys
  13 import os
  14 import time
  15 import traceback
  16 import shutil
  17 import operator
  18 import optparse
  19 from datetime import datetime
  20
  21 _valid_svn_actions = "MARD"   # The list of known SVN action abbr's, from "svn log"
  22
  23 # Module-level variables/parameters
  24 source_url = ""          # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
  25 source_repos_url = ""    # URL to root of source SVN repo,        e.g. 'http://server/svn/source'
  26 source_base = ""         # Relative path of source_url in source SVN repo, e.g. '/trunk'
  27 source_repos_uuid = ""   # UUID of source SVN repo
  28 target_url =""           # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
  29 rev_map = {}             # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
  30 options = None           # optparser options
  31
  32 def commit_from_svn_log_entry(log_entry, commit_paths=None, target_revprops=None):
  33     """
  34     Given an SVN log entry and an optional list of changed paths, do an svn commit.
  35     """
  36     # TODO: Run optional external shell hook here, for doing pre-commit filtering
  37     # Display the _wc_target "svn status" info if running in -vv (or higher) mode
  38     if ui.get_level() >= ui.EXTRA:
  39         ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
  40         ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
  41     # This will use the local timezone for displaying commit times
  42     timestamp = int(log_entry['date'])
  43     svn_date = str(datetime.fromtimestamp(timestamp))
  44     # Uncomment this one one if you prefer UTC commit times
  45     #svn_date = "%d 0" % timestamp
  46     args = ["commit", "--force-log"]
  47     if options.keep_author:
  48         args += ["-m", log_entry['message'] + "\nDate: " + svn_date, "--username", log_entry['author']]
  49     else:
  50         args += ["-m", log_entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + log_entry['author']]
  51     revprops = {}
  52     if log_entry['revprops']:
  53         # Carry forward any revprop's from the source revision
  54         for v in log_entry['revprops']:
  55             revprops[v['name']] = v['value']
  56     if target_revprops:
  57         # Add any extra revprop's we want to set for the target repo commits
  58         for v in target_revprops:
  59             revprops[v['name']] = v['value']
  60     if revprops:
  61         for key in revprops:
  62             args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
  63     if commit_paths:
  64         if len(commit_paths)<100:
  65             # If we don't have an excessive amount of individual changed paths, pass
  66             # those to the "svn commit" command. Else, pass nothing so we commit at
  67             # the root of the working-copy.
  68             args += list(commit_paths)
  69     rev = None
  70     if not options.dry_run:
  71         # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
  72         output = run_svn(args)
  73         if output:
  74             output_lines = output.strip("\n").split("\n")
  75             rev = ""
  76             for line in output_lines:
  77                 if line[0:19] == 'Committed revision ':
  78                     rev = line[19:].rstrip('.')
  79                     break
  80             if rev:
  81                 ui.status("Committed revision %s.", rev)
  82     return rev
  83
  84 def full_svn_revert():
  85     """
  86     Do an "svn revert" and proactively remove any extra files in the working copy.
  87     """
  88     run_svn(["revert", "--recursive", "."])
  89     output = run_svn(["status"])
  90     if output:
  91         output_lines = output.strip("\n").split("\n")
  92         for line in output_lines:
  93             if line[0] == "?":
  94                 path = line[4:].strip(" ")
  95                 if os.path.isfile(path):
  96                     os.remove(path)
  97                 if os.path.isdir(path):
  98                     shutil.rmtree(path)
  99
 100 def gen_tracking_revprops(source_rev):
 101     """
 102     Build an array of svn2svn-specific source-tracking revprops.
 103     """
 104     revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
 105                 {'name':'svn2svn:source_url',  'value':source_url},
 106                 {'name':'svn2svn:source_rev',  'value':source_rev}]
 107     return revprops
 108
 109 def sync_svn_props(source_url, source_rev, path_offset):
 110     """
 111     Carry-forward any unversioned properties from the source repo to the
 112     target WC.
 113     """
 114     source_props = svnclient.get_all_props(source_url+"/"+path_offset,  source_rev)
 115     target_props = svnclient.get_all_props(path_offset)
 116     if 'svn:mergeinfo' in source_props:
 117         # Never carry-forward "svn:mergeinfo"
 118         del source_props['svn:mergeinfo']
 119     for prop in target_props:
 120         if prop not in source_props:
 121             # Remove any properties which exist in target but not source
 122             run_svn(["propdel", prop, path_offset])
 123     for prop in source_props:
 124         if prop not in target_props or \
 125            source_props[prop] != target_props[prop]:
 126             # Set/update any properties which exist in source but not target or
 127             # whose value differs between source vs. target.
 128             run_svn(["propset", prop, source_props[prop], path_offset])
 129
 130 def in_svn(p, require_in_repo=False, prefix=""):
 131     """
 132     Check if a given file/folder is being tracked by Subversion.
 133     Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
 134     With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
 135     Use "svn status" to check the status of the file/folder.
 136     """
 137     entries = svnclient.get_svn_status(p, no_recursive=True)
 138     if not entries:
 139         return False
 140     d = entries[0]
 141     if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
 142         # If caller requires this path to be in the SVN repo, prevent returning True
 143         # for paths that are only locally-added.
 144         ret = False
 145     else:
 146         # Don't consider files tracked as deleted in the WC as under source-control.
 147         # Consider files which are locally added/copied as under source-control.
 148         ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
 149     ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
 150     return ret
 151
 152 def is_child_path(path, p_path):
 153     return True if (path == p_path) or (path.startswith(p_path+"/")) else False
 154
 155 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
 156     """
 157     Given a source path, walk the SVN history backwards to inspect the ancestory of
 158     that path, seeing if it traces back to base_path.  Build an array of copyfrom_path
 159     and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
 160     which base_path is a substring match of (e.g. we crawled back to the initial branch-
 161     copy from trunk), then return the collection of ancestor paths.  Otherwise,
 162     copyfrom_path has no ancestory compared to base_path.
 163
 164     This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
 165     file/folder was renamed in a branch and then that branch was merged back to trunk.
 166
 167     'svn_repos_url' is the full URL to the root of the SVN repository,
 168       e.g. 'file:///path/to/repo'
 169     'base_path' is the path in the SVN repo to the target path we're trying to
 170       trace ancestry back to, e.g. 'trunk'.
 171     'source_path' is the path in the SVN repo to the source path to start checking
 172       ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
 173       (full_path = svn_repos_url+base_path+"/"+path_offset)
 174     'source_rev' is the revision to start walking the history of source_path backwards from.
 175     """
 176     ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s  base_path: %s",
 177         svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
 178     done = False
 179     working_path = base_path+"/"+source_path
 180     working_rev  = source_rev
 181     first_iter_done = False
 182     ancestors_temp = []
 183     while not done:
 184         # Get the first "svn log" entry for this path (relative to @rev)
 185         ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
 186         log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path, 1, working_rev, True)
 187         if not log_entry:
 188             ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
 189             done = True
 190             break
 191         # If we found a copy-from case which matches our base_path, we're done.
 192         # ...but only if we've at least tried to search for the first copy-from path.
 193         if first_iter_done and is_child_path(working_path, base_path):
 194             ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(working_path, base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
 195             done = True
 196             break
 197         first_iter_done = True
 198         # Search for any actions on our target path (or parent paths).
 199         changed_paths_temp = []
 200         for d in log_entry['changed_paths']:
 201             path = d['path']
 202             if path in working_path:
 203                 changed_paths_temp.append({'path': path, 'data': d})
 204         if not changed_paths_temp:
 205             # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
 206             ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
 207             done = True
 208             continue
 209         # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
 210         changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
 211         # Find the action for our working_path in this revision. Use a loop to check in reverse order,
 212         # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
 213         for v in changed_paths:
 214             d = v['data']
 215             path = d['path']
 216             # Check action-type for this file
 217             action = d['action']
 218             if action not in _valid_svn_actions:
 219                 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 220                     % (log_entry['revision'], action))
 221             ui.status(prefix + "> %s %s%s", action, path,
 222                 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 223                 level=ui.DEBUG, color='YELLOW')
 224             if action == 'D':
 225                 # If file/folder was deleted, it has no ancestor
 226                 ancestors_temp = []
 227                 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
 228                 done = True
 229                 break
 230             if action in 'RA':
 231                 # If file/folder was added/replaced but not a copy, it has no ancestor
 232                 if not d['copyfrom_path']:
 233                     ancestors_temp = []
 234                     ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
 235                         "Added" if action == "A" else "Replaced",
 236                         level=ui.DEBUG, color='YELLOW')
 237                     done = True
 238                     break
 239                 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
 240                 # and keep checking for ancestors
 241                 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
 242                     action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
 243                     level=ui.DEBUG, color='YELLOW')
 244                 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
 245                                        'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
 246                 working_path = working_path.replace(d['path'], d['copyfrom_path'])
 247                 working_rev =  d['copyfrom_revision']
 248                 # Follow the copy and keep on searching
 249                 break
 250     ancestors = []
 251     if ancestors_temp:
 252         ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
 253         working_path = base_path+"/"+source_path
 254         for idx in range(len(ancestors_temp)):
 255             d = ancestors_temp[idx]
 256             working_path = working_path.replace(d['path'], d['copyfrom_path'])
 257             working_rev =  d['copyfrom_rev']
 258             ancestors.append({'path': working_path, 'revision': working_rev})
 259         if ui.get_level() >= ui.DEBUG:
 260             max_len = 0
 261             for idx in range(len(ancestors)):
 262                 d = ancestors[idx]
 263                 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
 264             ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
 265             for idx in range(len(ancestors)-1):
 266                 d = ancestors[idx]
 267                 d_next = ancestors[idx+1]
 268                 ui.status(prefix + " [%s] %s <-- %s", idx,
 269                     str(d['path']+"@"+str(d['revision'])).ljust(max_len),
 270                     str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
 271                     level=ui.DEBUG, color='YELLOW')
 272     else:
 273         ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
 274             svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
 275     return ancestors
 276
 277 def get_rev_map(source_rev, prefix):
 278     """
 279     Find the equivalent rev # in the target repo for the given rev # from the source repo.
 280     """
 281     ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
 282     # Find the highest entry less-than-or-equal-to source_rev
 283     for rev in range(int(source_rev), 0, -1):
 284         ui.status(prefix + ">> get_rev_map: rev=%s  in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
 285         if rev in rev_map:
 286             return int(rev_map[rev])
 287     # Else, we fell off the bottom of the rev_map. Ruh-roh...
 288     return None
 289
 290 def set_rev_map(source_rev, target_rev):
 291     ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
 292     global rev_map
 293     rev_map[int(source_rev)]=int(target_rev)
 294
 295 def build_rev_map(target_url, target_end_rev, source_info):
 296     """
 297     Check for any already-replayed history from source_url (source_info) and
 298     build the mapping-table of source_rev -> target_rev.
 299     """
 300     global rev_map
 301     rev_map = {}
 302     ui.status("Rebuilding rev_map...", level=ui.VERBOSE)
 303     proc_count = 0
 304     it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
 305     for log_entry in it_log_entries:
 306         if log_entry['revprops']:
 307             revprops = {}
 308             for v in log_entry['revprops']:
 309                 if v['name'].startswith('svn2svn:'):
 310                     revprops[v['name']] = v['value']
 311             if revprops and \
 312                revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
 313                revprops['svn2svn:source_url'] == source_info['url']:
 314                 source_rev = revprops['svn2svn:source_rev']
 315                 target_rev = log_entry['revision']
 316                 set_rev_map(source_rev, target_rev)
 317
 318 def get_svn_dirlist(svn_path, svn_rev = ""):
 319     """
 320     Get a list of all the child contents (recusive) of the given folder path.
 321     """
 322     args = ["list"]
 323     path = svn_path
 324     if svn_rev:
 325         args += ["-r", svn_rev]
 326         path += "@"+str(svn_rev)
 327     args += [path]
 328     paths = run_svn(args, no_fail=True)
 329     paths = paths.strip("\n").split("\n") if len(paths)>1 else []
 330     return paths
 331
 332 def path_in_list(paths, path):
 333     for p in paths:
 334         if is_child_path(path, p):
 335             return True
 336     return False
 337
 338 def add_path(paths, path):
 339     if not path_in_list(paths, path):
 340         paths.append(path)
 341
 342 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
 343                export_paths={}, is_dir = False, skip_paths=[], prefix = ""):
 344     """
 345     Given the add'd source path, replay the "svn add/copy" commands to correctly
 346     track renames across copy-from's.
 347
 348     For example, consider a sequence of events like this:
 349     1. svn copy /trunk /branches/fix1
 350     2. (Make some changes on /branches/fix1)
 351     3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2  " Rename folder
 352     4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt  " Rename file inside renamed folder
 353     5. svn co /trunk && svn merge /branches/fix1
 354     After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
 355     and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
 356     to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
 357     we'd lose the logical history that Proj2/file2.txt is really a descendant
 358     of Proj1/file1.txt.
 359
 360     'path_offset' is the offset from source_base to the file to check ancestry for,
 361       e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
 362     'source_rev' is the revision ("svn log") that we're processing from the source repo.
 363     'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
 364       directory, when being called recursively by do_svn_add_dir().
 365     'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
 366     'is_dir' is whether path_offset is a directory (rather than a file).
 367     """
 368     ui.status(prefix + ">> do_svn_add: %s  %s", source_base+"/"+path_offset+"@"+str(source_rev),
 369         "  (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
 370         level=ui.DEBUG, color='GREEN')
 371     # Check if the given path has ancestors which chain back to the current source_base
 372     found_ancestor = False
 373     ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+"  ")
 374     # ancestors[n] is the original (pre-branch-copy) trunk path.
 375     # ancestors[n-1] is the first commit on the new branch.
 376     copyfrom_path = ancestors[len(ancestors)-1]['path']     if ancestors else ""
 377     copyfrom_rev  = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
 378     if ancestors:
 379         # The copy-from path has ancestory back to source_url.
 380         ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
 381             level=ui.DEBUG, color='GREEN', bold=True)
 382         found_ancestor = True
 383         # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
 384         # return None in the case where copyfrom_rev is *before* our source_start_rev.
 385         tgt_rev = get_rev_map(copyfrom_rev, prefix+"  ")
 386         ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
 387     else:
 388         ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
 389         found_ancestor = False
 390     if found_ancestor and tgt_rev:
 391         # Check if this path_offset in the target WC already has this ancestry, in which
 392         # case there's no need to run the "svn copy" (again).
 393         path_in_svn = in_svn(path_offset, prefix=prefix+"  ")
 394         log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+"  ") else []
 395         if (not log_entry or (log_entry['revision'] != tgt_rev)):
 396             copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
 397             ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 398             ui.status(prefix + "   copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 399             ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
 400             if path_in_svn and \
 401                ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
 402                 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
 403                 # When being called recursively, if this child entry has the same ancestor as the
 404                 # the parent, then no need to try to run another "svn copy".
 405                 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
 406                     parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
 407                 pass
 408             else:
 409                 # Copy this path from the equivalent path+rev in the target repo, to create the
 410                 # equivalent history.
 411                 if parent_copyfrom_path:
 412                     # If we have a parent copy-from path, we mis-match that so display a status
 413                     # message describing the action we're mimic'ing. If path_in_svn, then this
 414                     # is logically a "replace" rather than an "add".
 415                     ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
 416                 if path_in_svn:
 417                     # If local file is already under version-control, then this is a replace.
 418                     ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
 419                     run_svn(["remove", "--force", path_offset])
 420                 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
 421                 if is_dir:
 422                     # Export the final verison of all files in this folder.
 423                     add_path(export_paths, path_offset)
 424                 else:
 425                     # Export the final verison of this file.
 426                     run_svn(["export", "--force", "-r", source_rev,
 427                              source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 428                 # Copy SVN properties from source repo
 429                 sync_svn_props(source_url, source_rev, path_offset)
 430         else:
 431             ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
 432     else:
 433         # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
 434         # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
 435         # Create (parent) directory if needed.
 436         # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
 437         #       split-out to a shared tag?
 438         p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
 439         if not os.path.exists(p_path):
 440             run_svn(["mkdir", p_path])
 441         if not in_svn(path_offset, prefix=prefix+"  "):
 442             if is_dir:
 443                 # Export the final verison of all files in this folder.
 444                 add_path(export_paths, path_offset)
 445             else:
 446                 # Export the final verison of this file. We *need* to do this before running
 447                 # the "svn add", even if we end-up re-exporting this file again via export_paths.
 448                 run_svn(["export", "--force", "-r", source_rev,
 449                          source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 450             # If not already under version-control, then "svn add" this file/folder.
 451             run_svn(["add", "--parents", path_offset])
 452         # Copy SVN properties from source repo
 453         sync_svn_props(source_url, source_rev, path_offset)
 454     if is_dir:
 455         # For any folders that we process, process any child contents, so that we correctly
 456         # replay copies/replaces/etc.
 457         do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, skip_paths, prefix+"  ")
 458
 459 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
 460                    export_paths, skip_paths, prefix=""):
 461     # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
 462     # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
 463     #       associated remote repo folder. (Is this a problem?)
 464     paths_local =  get_svn_dirlist(path_offset)
 465     paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
 466     ui.status(prefix + ">> do_svn_add_dir: paths_local:  %s", str(paths_local),  level=ui.DEBUG, color='GREEN')
 467     ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
 468     # Update files/folders which exist in remote but not local
 469     for path in paths_remote:
 470         path_is_dir = True if path[-1] == "/" else False
 471         working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
 472         if not working_path in skip_paths:
 473             do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
 474                        export_paths, path_is_dir, skip_paths, prefix+"  ")
 475     # Remove files/folders which exist in local but not remote
 476     for path in paths_local:
 477         if not path in paths_remote:
 478             ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
 479             run_svn(["remove", "--force", path_offset+"/"+path])
 480             # TODO: Does this handle deleted folders too? Wouldn't want to have a case
 481             #       where we only delete all files from folder but leave orphaned folder around.
 482
 483 def process_svn_log_entry(log_entry, commit_paths, prefix = ""):
 484     """
 485     Process SVN changes from the given log entry. Build an array (commit_paths)
 486     of the paths in the working-copy that were changed, i.e. the paths which
 487     we'll pass to "svn commit".
 488     """
 489     export_paths = []
 490     source_rev = log_entry['revision']
 491     ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
 492     for d in log_entry['changed_paths']:
 493         # Get the full path for this changed_path
 494         # e.g. '/branches/bug123/projectA/file1.txt'
 495         path = d['path']
 496         if not is_child_path(path, source_base):
 497             # Ignore changed files that are not part of this subdir
 498             ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s  (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
 499             continue
 500         if d['kind'] == "":
 501             # The "kind" value was introduced in SVN 1.6, and "svn log --xml" won't return a "kind"
 502             # value for commits made on a pre-1.6 repo, even if the server is now running 1.6.
 503             # We need to use other methods to fetch the node-kind for these cases.
 504             d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
 505         assert (d['kind'] == 'file') or (d['kind'] == 'dir')
 506         path_is_dir =  True if d['kind'] == 'dir'  else False
 507         path_is_file = True if d['kind'] == 'file' else False
 508         # Calculate the offset (based on source_base) for this changed_path
 509         # e.g. 'projectA/file1.txt'
 510         # (path = source_base + "/" + path_offset)
 511         path_offset = path[len(source_base):].strip("/")
 512         # Get the action for this path
 513         action = d['action']
 514         if action not in _valid_svn_actions:
 515             raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 516                 % (source_rev, action))
 517         ui.status(" %s %s%s", action, d['path'],
 518             (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 519             level=ui.VERBOSE)
 520
 521         # Try to be efficient and keep track of an explicit list of paths in the
 522         # working copy that changed. If we commit from the root of the working copy,
 523         # then SVN needs to crawl the entire working copy looking for pending changes.
 524         commit_paths.append(path_offset)
 525
 526         # Special-handling for replace's
 527         if action == 'R':
 528             # If file was "replaced" (deleted then re-added, all in same revision),
 529             # then we need to run the "svn rm" first, then change action='A'. This
 530             # lets the normal code below handle re-"svn add"'ing the files. This
 531             # should replicate the "replace".
 532             if in_svn(path_offset):
 533                 # Target path might not be under version-control yet, e.g. parent "add"
 534                 # was a copy-from a branch which had no ancestry back to trunk, and each
 535                 # child folder under that parent folder is a "replace" action on the final
 536                 # merge to trunk. Since the child folders will be in skip_paths, do_svn_add
 537                 # wouldn't have created them while processing the parent "add" path.
 538                 run_svn(["remove", "--force", path_offset])
 539             action = 'A'
 540
 541         # Handle all the various action-types
 542         # (Handle "add" first, for "svn copy/move" support)
 543         if action == 'A':
 544             # Determine where to export from.
 545             svn_copy = False
 546             # Handle cases where this "add" was a copy from another URL in the source repo
 547             if d['copyfrom_revision']:
 548                 copyfrom_path = d['copyfrom_path']
 549                 copyfrom_rev =  d['copyfrom_revision']
 550                 skip_paths = []
 551                 for tmp_d in log_entry['changed_paths']:
 552                     tmp_path = tmp_d['path']
 553                     if is_child_path(tmp_path, path):
 554                         # Build list of child entries which are also in the changed_paths list,
 555                         # so that do_svn_add() can skip processing these entries when recursing
 556                         # since we'll end-up processing them later.
 557                         tmp_path_offset = tmp_path[len(source_base):].strip("/")
 558                         skip_paths.append(tmp_path_offset)
 559                 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, skip_paths, prefix+"  ")
 560             # Else just "svn export" the files from the source repo and "svn add" them.
 561             else:
 562                 # Create (parent) directory if needed
 563                 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
 564                 if not os.path.exists(p_path):
 565                     run_svn(["mkdir", p_path])
 566                 # Export the entire added tree.
 567                 if path_is_dir:
 568                     # For directories, defer the (recurisve) "svn export". Might have a
 569                     # situation in a branch merge where the entry in the svn-log is a
 570                     # non-copy-from'd "add" but there are child contents (that we haven't
 571                     # gotten to yet in log_entry) that are copy-from's.  When we try do
 572                     # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
 573                     # having pre-existing (svn-add'd) contents creates some trouble.
 574                     # Instead, just create the stub folders ("svn mkdir" above) and defer
 575                     # exporting the final file-state until the end.
 576                     add_path(export_paths, path_offset)
 577                 else:
 578                     # Export the final verison of this file. We *need* to do this before running
 579                     # the "svn add", even if we end-up re-exporting this file again via export_paths.
 580                     run_svn(["export", "--force", "-r", source_rev,
 581                              source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 582                 if not in_svn(path_offset, prefix=prefix+"  "):
 583                     # Need to use in_svn here to handle cases where client committed the parent
 584                     # folder and each indiv sub-folder.
 585                     run_svn(["add", "--parents", path_offset])
 586                 # Copy SVN properties from source repo
 587                 sync_svn_props(source_url, source_rev, path_offset)
 588
 589         elif action == 'D':
 590             run_svn(["remove", "--force", path_offset])
 591
 592         elif action == 'M':
 593             if path_is_file:
 594                 run_svn(["export", "--force", "-N" , "-r", source_rev,
 595                          source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 596             # Update SVN properties based on source repo
 597             sync_svn_props(source_url, source_rev, path_offset)
 598
 599         else:
 600             raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
 601                 % action)
 602
 603     # Export the final version of all add'd paths from source_url
 604     if export_paths:
 605         for path_offset in export_paths:
 606             run_svn(["export", "--force", "-r", source_rev,
 607                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 608
 609 def disp_svn_log_summary(log_entry):
 610     ui.status("")
 611     ui.status("r%s | %s | %s",
 612         log_entry['revision'],
 613         log_entry['author'],
 614         str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
 615     ui.status(log_entry['message'])
 616     ui.status("------------------------------------------------------------------------")
 617
 618 def real_main(args, parser):
 619     global source_url, target_url, rev_map
 620     source_url = args.pop(0).rstrip("/")    # e.g. 'http://server/svn/source/trunk'
 621     target_url = args.pop(0).rstrip("/")    # e.g. 'file:///svn/target/trunk'
 622     ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
 623
 624     # Make sure that both the source and target URL's are valid
 625     source_info = svnclient.get_svn_info(source_url)
 626     assert is_child_path(source_url, source_info['repos_url'])
 627     target_info = svnclient.get_svn_info(target_url)
 628     assert is_child_path(target_url, target_info['repos_url'])
 629
 630     # Init global vars
 631     global source_repos_url,source_base,source_repos_uuid
 632     source_repos_url = source_info['repos_url']       # e.g. 'http://server/svn/source'
 633     source_base = source_url[len(source_repos_url):]  # e.g. '/trunk'
 634     source_repos_uuid = source_info['repos_uuid']
 635
 636     # Init start and end revision
 637     source_start_rev = svnclient.get_svn_rev(source_repos_url, options.svn_rev_start if options.svn_rev_start else 1)
 638     source_end_rev   = svnclient.get_svn_rev(source_repos_url, options.svn_rev_end   if options.svn_rev_end   else "HEAD")
 639
 640     target_end_rev = target_info['revision']   # Last revision # in the target repo
 641     wc_target = os.path.abspath('_wc_target')
 642     num_entries_proc = 0
 643     commit_count = 0
 644     source_rev = None
 645     target_rev = None
 646
 647     # Check out a working copy of target_url if needed
 648     wc_exists = os.path.exists(wc_target)
 649     if wc_exists and not options.cont_from_break:
 650         shutil.rmtree(wc_target)
 651         wc_exists = False
 652     if not wc_exists:
 653         ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
 654         svnclient.svn_checkout(target_url, wc_target)
 655     os.chdir(wc_target)
 656
 657     if not options.cont_from_break:
 658         # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
 659         # Get the first log entry at/after source_start_rev, which is where
 660         # we'll do the initial import from.
 661         it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False)
 662         for source_start_log in it_log_start:
 663             break
 664         if not source_start_log:
 665             raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
 666                 (source_start_rev, source_end_rev, source_url))
 667
 668         # This is the revision we will start from for source_url
 669         source_start_rev = source_rev = int(source_start_log['revision'])
 670         ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
 671
 672         # For the initial commit to the target URL, export all the contents from
 673         # the source URL at the start-revision.
 674         disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
 675         ui.status("(Initial import)", level=ui.VERBOSE)
 676         # Export and add file-contents from source_url@source_start_rev
 677         top_paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
 678         top_paths = top_paths.strip("\n").split("\n")
 679         for path in top_paths:
 680             # For each top-level file/folder...
 681             if not path:
 682                 continue
 683             # Directories have a trailing slash in the "svn list" output
 684             path_is_dir = True if path[-1] == "/" else False
 685             path_offset = path.rstrip('/') if path_is_dir else path
 686             if in_svn(path_offset, prefix="  "):
 687                 raise InternalError("Cannot replay history on top of pre-existing structure: %s" % source_url+"/"+path_offset)
 688             if path_is_dir and not os.path.exists(path_offset):
 689                 os.makedirs(path_offset)
 690             run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 691             run_svn(["add", path_offset])
 692         # Update any properties on the newly added content
 693         paths = run_svn(["list", "--recursive", "-r", source_rev, source_url+"@"+str(source_rev)])
 694         paths = paths.strip("\n").split("\n")
 695         sync_svn_props(source_url, source_rev, "")
 696         for path in paths:
 697             if not path:
 698                 continue
 699             # Directories have a trailing slash in the "svn list" output
 700             path_is_dir = True if path[-1] == "/" else False
 701             path_offset = path.rstrip('/') if path_is_dir else path
 702             ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path_offset, level=ui.VERBOSE)
 703             sync_svn_props(source_url, source_rev, path_offset)
 704         # Commit the initial import
 705         num_entries_proc += 1
 706         target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 707         target_rev = commit_from_svn_log_entry(source_start_log, target_revprops=target_revprops)
 708         if target_rev:
 709             # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 710             set_rev_map(source_rev, target_rev)
 711             # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 712             run_svn(["update"])
 713             commit_count += 1
 714     else:
 715         # Re-build the rev_map based on any already-replayed history in target_url
 716         build_rev_map(target_url, target_end_rev, source_info)
 717         if not rev_map:
 718             raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url)
 719         source_start_rev = int(max(rev_map, key=rev_map.get))
 720         assert source_start_rev
 721         ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
 722
 723     svn_vers_t = svnclient.get_svn_client_version()
 724     svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
 725
 726     # Load SVN log starting from source_start_rev + 1
 727     it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else []
 728     source_rev = None
 729
 730     try:
 731         for log_entry in it_log_entries:
 732             if options.entries_proc_limit:
 733                 if num_entries_proc >= options.entries_proc_limit:
 734                     break
 735             # Replay this revision from source_url into target_url
 736             disp_svn_log_summary(log_entry)
 737             source_rev = log_entry['revision']
 738             # Process all the changed-paths in this log entry
 739             commit_paths = []
 740             process_svn_log_entry(log_entry, commit_paths)
 741             num_entries_proc += 1
 742             # Commit any changes made to _wc_target
 743             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 744             target_rev = commit_from_svn_log_entry(log_entry, commit_paths, target_revprops=target_revprops)
 745             if target_rev:
 746                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 747                 source_rev = log_entry['revision']
 748                 set_rev_map(source_rev, target_rev)
 749                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 750                 run_svn(["update"])
 751                 commit_count += 1
 752                 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
 753                 if svn_vers >= 1.7 and (commit_count % 100 == 0):
 754                     run_svn(["cleanup"])
 755         if not source_rev:
 756             # If there were no new source_url revisions to process, init source_rev
 757             # for the "finally" message below.
 758             source_rev = source_end_rev
 759
 760     except KeyboardInterrupt:
 761         print "\nStopped by user."
 762         print "\nCleaning-up..."
 763         run_svn(["cleanup"])
 764         full_svn_revert()
 765     except:
 766         print "\nCommand failed with following error:\n"
 767         traceback.print_exc()
 768         print "\nCleaning-up..."
 769         run_svn(["cleanup"])
 770         print run_svn(["status"])
 771         full_svn_revert()
 772     finally:
 773         print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
 774
 775 def main():
 776     # Defined as entry point. Must be callable without arguments.
 777     usage = "Usage: %prog [OPTIONS] source_url target_url"
 778     description = """\
 779   Replicate (replay) history from one SVN repository to another. Maintain
 780   logical ancestry wherever possible, so that 'svn log' on the replayed
 781   repo will correctly follow file/folder renames.
 782
 783   == Examples ==
 784   Create a copy of only /trunk from source repo, starting at r5000
 785   $ svnadmin create /svn/target
 786   $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
 787   $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
 788     1. The target_url will be checked-out to ./_wc_target
 789     2. The first commit to http://server/source/trunk at/after r5000 will be
 790        exported & added into _wc_target
 791     3. All revisions affecting http://server/source/trunk (starting at r5000)
 792        will be replayed to _wc_target. Any add/copy/move/replaces that are
 793        copy-from'd some path outside of /trunk (e.g. files renamed on a
 794        /branch and branch was merged into /trunk) will correctly maintain
 795        logical ancestry where possible.
 796
 797   Use continue-mode (-c) to pick-up where the last run left-off
 798   $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
 799     1. The target_url will be checked-out to ./_wc_target, if not already
 800        checked-out
 801     2. All new revisions affecting http://server/source/trunk starting from
 802        the last replayed revision to file:///svn/target/trunk (based on the
 803        svn2svn:* revprops) will be replayed to _wc_target, maintaining all
 804        logical ancestry where possible."""
 805     parser = optparse.OptionParser(usage, description=description,
 806                 formatter=HelpFormatter(), version="%prog "+str(full_version))
 807     #parser.remove_option("--help")
 808     #parser.add_option("-h", "--help", dest="show_help", action="store_true",
 809     #    help="show this help message and exit")
 810     parser.add_option("-r", "--revision", type="string", dest="svn_rev", metavar="ARG",
 811                       help="revision range to replay from source_url\n" + \
 812                            "A revision argument can be one of:\n" + \
 813                            "   START        start rev # (end will be 'HEAD')\n" + \
 814                            "   START:END    start and ending rev #'s\n" + \
 815                            "(Any revision # formats which SVN understands\n" + \
 816                            " are supported, e.g. 'HEAD', '{2010-01-31}', etc.)")
 817     parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", default=False,
 818                       help="maintain original 'Author' info from source repo")
 819     parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
 820                       help="continue from previous break")
 821     parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
 822                       help="maximum number of log entries to process")
 823     parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 824                       help="try processing next log entry but don't commit changes to "
 825                            "target working-copy (forces --limit=1)")
 826     parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
 827                       help="enable additional output (use -vv or -vvv for more)")
 828     parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
 829                       help="enable debugging output (same as -vvv)")
 830     global options
 831     options, args = parser.parse_args()
 832     if len(args) != 2:
 833         parser.error("incorrect number of arguments")
 834     if options.verbosity < 10:
 835         # Expand multiple "-v" arguments to a real ui._level value
 836         options.verbosity *= 10
 837     if options.dry_run:
 838         # When in dry-run mode, only try to process the next log_entry
 839         options.entries_proc_limit = 1
 840     options.svn_rev_start = None
 841     options.svn_rev_end   = None
 842     if options.svn_rev:
 843         rev = options.svn_rev.split(":")
 844         options.svn_rev_start = rev[0] if len(rev)>0 else None
 845         options.svn_rev_end   = rev[1] if len(rev)>1 else None
 846     ui.update_config(options)
 847     return real_main(args, parser)
 848
 849
 850 if __name__ == "__main__":
 851     sys.exit(main() or 0)