svn2svn/run/svn2svn.py

   1 """
   2 Replicate (replay) changesets from one SVN repository to another.
   3 """
   4
   5 from .. import base_version, full_version
   6 from .. import ui
   7 from .. import svnclient
   8 from ..shell import run_svn
   9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
  10 from parse import HelpFormatter
  11
  12 import sys
  13 import os
  14 import time
  15 import traceback
  16 import shutil
  17 import operator
  18 import optparse
  19 import re
  20 from datetime import datetime
  21
  22 _valid_svn_actions = "MARD"   # The list of known SVN action abbr's, from "svn log"
  23
  24 # Module-level variables/parameters
  25 source_url = ""          # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
  26 source_repos_url = ""    # URL to root of source SVN repo,        e.g. 'http://server/svn/source'
  27 source_base = ""         # Relative path of source_url in source SVN repo, e.g. '/trunk'
  28 source_repos_uuid = ""   # UUID of source SVN repo
  29 target_url =""           # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
  30 rev_map = {}             # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
  31 options = None           # optparser options
  32
  33 def commit_from_svn_log_entry(log_entry, commit_paths=None, target_revprops=None):
  34     """
  35     Given an SVN log entry and an optional list of changed paths, do an svn commit.
  36     """
  37     # TODO: Run optional external shell hook here, for doing pre-commit filtering
  38     # Display the _wc_target "svn status" info if running in -vv (or higher) mode
  39     if ui.get_level() >= ui.EXTRA:
  40         ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
  41         ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
  42     # This will use the local timezone for displaying commit times
  43     timestamp = int(log_entry['date'])
  44     svn_date = str(datetime.fromtimestamp(timestamp))
  45     # Uncomment this one one if you prefer UTC commit times
  46     #svn_date = "%d 0" % timestamp
  47     args = ["commit", "--force-log"]
  48     if options.keep_author:
  49         args += ["-m", log_entry['message'] + "\nDate: " + svn_date, "--username", log_entry['author']]
  50     else:
  51         args += ["-m", log_entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + log_entry['author']]
  52     revprops = {}
  53     if log_entry['revprops']:
  54         # Carry forward any revprop's from the source revision
  55         for v in log_entry['revprops']:
  56             revprops[v['name']] = v['value']
  57     if target_revprops:
  58         # Add any extra revprop's we want to set for the target repo commits
  59         for v in target_revprops:
  60             revprops[v['name']] = v['value']
  61     if revprops:
  62         for key in revprops:
  63             args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
  64     if commit_paths:
  65         if len(commit_paths)<100:
  66             # If we don't have an excessive amount of individual changed paths, pass
  67             # those to the "svn commit" command. Else, pass nothing so we commit at
  68             # the root of the working-copy.
  69             args += list(commit_paths)
  70     rev = None
  71     if not options.dry_run:
  72         # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
  73         output = run_svn(args)
  74         if output:
  75             output_lines = output.strip("\n").split("\n")
  76             rev = ""
  77             for line in output_lines:
  78                 if line[0:19] == 'Committed revision ':
  79                     rev = line[19:].rstrip('.')
  80                     break
  81             if rev:
  82                 ui.status("Committed revision %s.", rev)
  83     return rev
  84
  85 def full_svn_revert():
  86     """
  87     Do an "svn revert" and proactively remove any extra files in the working copy.
  88     """
  89     run_svn(["revert", "--recursive", "."])
  90     output = run_svn(["status"])
  91     if output:
  92         output_lines = output.strip("\n").split("\n")
  93         for line in output_lines:
  94             if line[0] == "?":
  95                 path = line[4:].strip(" ")
  96                 if os.path.isfile(path):
  97                     os.remove(path)
  98                 if os.path.isdir(path):
  99                     shutil.rmtree(path)
 100
 101 def gen_tracking_revprops(source_rev):
 102     """
 103     Build an array of svn2svn-specific source-tracking revprops.
 104     """
 105     revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
 106                 {'name':'svn2svn:source_url',  'value':source_url},
 107                 {'name':'svn2svn:source_rev',  'value':source_rev}]
 108     return revprops
 109
 110 def sync_svn_props(source_url, source_rev, path_offset):
 111     """
 112     Carry-forward any unversioned properties from the source repo to the
 113     target WC.
 114     """
 115     source_props = svnclient.get_all_props(source_url+"/"+path_offset,  source_rev)
 116     target_props = svnclient.get_all_props(path_offset)
 117     if 'svn:mergeinfo' in source_props:
 118         # Never carry-forward "svn:mergeinfo"
 119         del source_props['svn:mergeinfo']
 120     for prop in target_props:
 121         if prop not in source_props:
 122             # Remove any properties which exist in target but not source
 123             run_svn(["propdel", prop, path_offset])
 124     for prop in source_props:
 125         if prop not in target_props or \
 126            source_props[prop] != target_props[prop]:
 127             # Set/update any properties which exist in source but not target or
 128             # whose value differs between source vs. target.
 129             run_svn(["propset", prop, source_props[prop], path_offset])
 130
 131 def in_svn(p, require_in_repo=False, prefix=""):
 132     """
 133     Check if a given file/folder is being tracked by Subversion.
 134     Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
 135     With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
 136     Use "svn status" to check the status of the file/folder.
 137     """
 138     entries = svnclient.get_svn_status(p, no_recursive=True)
 139     if not entries:
 140         return False
 141     d = entries[0]
 142     if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
 143         # If caller requires this path to be in the SVN repo, prevent returning True
 144         # for paths that are only locally-added.
 145         ret = False
 146     else:
 147         # Don't consider files tracked as deleted in the WC as under source-control.
 148         # Consider files which are locally added/copied as under source-control.
 149         ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
 150     ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
 151     return ret
 152
 153 def is_child_path(path, p_path):
 154     return True if (path == p_path) or (path.startswith(p_path+"/")) else False
 155
 156 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
 157     """
 158     Given a source path, walk the SVN history backwards to inspect the ancestory of
 159     that path, seeing if it traces back to base_path.  Build an array of copyfrom_path
 160     and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
 161     which base_path is a substring match of (e.g. we crawled back to the initial branch-
 162     copy from trunk), then return the collection of ancestor paths.  Otherwise,
 163     copyfrom_path has no ancestory compared to base_path.
 164
 165     This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
 166     file/folder was renamed in a branch and then that branch was merged back to trunk.
 167
 168     'svn_repos_url' is the full URL to the root of the SVN repository,
 169       e.g. 'file:///path/to/repo'
 170     'base_path' is the path in the SVN repo to the target path we're trying to
 171       trace ancestry back to, e.g. 'trunk'.
 172     'source_path' is the path in the SVN repo to the source path to start checking
 173       ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
 174       (full_path = svn_repos_url+base_path+"/"+path_offset)
 175     'source_rev' is the revision to start walking the history of source_path backwards from.
 176     """
 177     ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s  base_path: %s",
 178         svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
 179     done = False
 180     working_path = base_path+"/"+source_path
 181     working_rev  = source_rev
 182     first_iter_done = False
 183     ancestors_temp = []
 184     while not done:
 185         # Get the first "svn log" entry for this path (relative to @rev)
 186         ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
 187         log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path, 1, working_rev, True)
 188         if not log_entry:
 189             ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
 190             done = True
 191             break
 192         # If we found a copy-from case which matches our base_path, we're done.
 193         # ...but only if we've at least tried to search for the first copy-from path.
 194         if first_iter_done and is_child_path(working_path, base_path):
 195             ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(working_path, base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
 196             done = True
 197             break
 198         first_iter_done = True
 199         # Search for any actions on our target path (or parent paths).
 200         changed_paths_temp = []
 201         for d in log_entry['changed_paths']:
 202             path = d['path']
 203             if path in working_path:
 204                 changed_paths_temp.append({'path': path, 'data': d})
 205         if not changed_paths_temp:
 206             # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
 207             ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
 208             done = True
 209             continue
 210         # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
 211         changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
 212         # Find the action for our working_path in this revision. Use a loop to check in reverse order,
 213         # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
 214         for v in changed_paths:
 215             d = v['data']
 216             path = d['path']
 217             # Check action-type for this file
 218             action = d['action']
 219             if action not in _valid_svn_actions:
 220                 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 221                     % (log_entry['revision'], action))
 222             ui.status(prefix + "> %s %s%s", action, path,
 223                 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 224                 level=ui.DEBUG, color='YELLOW')
 225             if action == 'D':
 226                 # If file/folder was deleted, it has no ancestor
 227                 ancestors_temp = []
 228                 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
 229                 done = True
 230                 break
 231             if action in 'RA':
 232                 # If file/folder was added/replaced but not a copy, it has no ancestor
 233                 if not d['copyfrom_path']:
 234                     ancestors_temp = []
 235                     ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
 236                         "Added" if action == "A" else "Replaced",
 237                         level=ui.DEBUG, color='YELLOW')
 238                     done = True
 239                     break
 240                 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
 241                 # and keep checking for ancestors
 242                 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
 243                     action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
 244                     level=ui.DEBUG, color='YELLOW')
 245                 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
 246                                        'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
 247                 working_path = working_path.replace(d['path'], d['copyfrom_path'])
 248                 working_rev =  d['copyfrom_revision']
 249                 # Follow the copy and keep on searching
 250                 break
 251     ancestors = []
 252     if ancestors_temp:
 253         ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
 254         working_path = base_path+"/"+source_path
 255         for idx in range(len(ancestors_temp)):
 256             d = ancestors_temp[idx]
 257             working_path = working_path.replace(d['path'], d['copyfrom_path'])
 258             working_rev =  d['copyfrom_rev']
 259             ancestors.append({'path': working_path, 'revision': working_rev})
 260         if ui.get_level() >= ui.DEBUG:
 261             max_len = 0
 262             for idx in range(len(ancestors)):
 263                 d = ancestors[idx]
 264                 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
 265             ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
 266             for idx in range(len(ancestors)-1):
 267                 d = ancestors[idx]
 268                 d_next = ancestors[idx+1]
 269                 ui.status(prefix + " [%s] %s <-- %s", idx,
 270                     str(d['path']+"@"+str(d['revision'])).ljust(max_len),
 271                     str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
 272                     level=ui.DEBUG, color='YELLOW')
 273     else:
 274         ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
 275             svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
 276     return ancestors
 277
 278 def get_rev_map(source_rev, prefix):
 279     """
 280     Find the equivalent rev # in the target repo for the given rev # from the source repo.
 281     """
 282     ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
 283     # Find the highest entry less-than-or-equal-to source_rev
 284     for rev in range(int(source_rev), 0, -1):
 285         ui.status(prefix + ">> get_rev_map: rev=%s  in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
 286         if rev in rev_map:
 287             return int(rev_map[rev])
 288     # Else, we fell off the bottom of the rev_map. Ruh-roh...
 289     return None
 290
 291 def set_rev_map(source_rev, target_rev):
 292     ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
 293     global rev_map
 294     rev_map[int(source_rev)]=int(target_rev)
 295
 296 def build_rev_map(target_url, target_end_rev, source_info):
 297     """
 298     Check for any already-replayed history from source_url (source_info) and
 299     build the mapping-table of source_rev -> target_rev.
 300     """
 301     global rev_map
 302     rev_map = {}
 303     ui.status("Rebuilding rev_map...", level=ui.VERBOSE)
 304     proc_count = 0
 305     it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
 306     for log_entry in it_log_entries:
 307         if log_entry['revprops']:
 308             revprops = {}
 309             for v in log_entry['revprops']:
 310                 if v['name'].startswith('svn2svn:'):
 311                     revprops[v['name']] = v['value']
 312             if revprops and \
 313                revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
 314                revprops['svn2svn:source_url'] == source_info['url']:
 315                 source_rev = revprops['svn2svn:source_rev']
 316                 target_rev = log_entry['revision']
 317                 set_rev_map(source_rev, target_rev)
 318
 319 def get_svn_dirlist(svn_path, svn_rev = ""):
 320     """
 321     Get a list of all the child contents (recusive) of the given folder path.
 322     """
 323     args = ["list"]
 324     path = svn_path
 325     if svn_rev:
 326         args += ["-r", svn_rev]
 327         path += "@"+str(svn_rev)
 328     args += [path]
 329     paths = run_svn(args, no_fail=True)
 330     paths = paths.strip("\n").split("\n") if len(paths)>1 else []
 331     return paths
 332
 333 def path_in_list(paths, path):
 334     for p in paths:
 335         if is_child_path(path, p):
 336             return True
 337     return False
 338
 339 def add_path(paths, path):
 340     if not path_in_list(paths, path):
 341         paths.append(path)
 342
 343 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
 344                export_paths={}, is_dir = False, skip_paths=[], prefix = ""):
 345     """
 346     Given the add'd source path, replay the "svn add/copy" commands to correctly
 347     track renames across copy-from's.
 348
 349     For example, consider a sequence of events like this:
 350     1. svn copy /trunk /branches/fix1
 351     2. (Make some changes on /branches/fix1)
 352     3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2  " Rename folder
 353     4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt  " Rename file inside renamed folder
 354     5. svn co /trunk && svn merge /branches/fix1
 355     After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
 356     and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
 357     to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
 358     we'd lose the logical history that Proj2/file2.txt is really a descendant
 359     of Proj1/file1.txt.
 360
 361     'path_offset' is the offset from source_base to the file to check ancestry for,
 362       e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
 363     'source_rev' is the revision ("svn log") that we're processing from the source repo.
 364     'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
 365       directory, when being called recursively by do_svn_add_dir().
 366     'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
 367     'is_dir' is whether path_offset is a directory (rather than a file).
 368     """
 369     ui.status(prefix + ">> do_svn_add: %s  %s", source_base+"/"+path_offset+"@"+str(source_rev),
 370         "  (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
 371         level=ui.DEBUG, color='GREEN')
 372     # Check if the given path has ancestors which chain back to the current source_base
 373     found_ancestor = False
 374     ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+"  ")
 375     # ancestors[n] is the original (pre-branch-copy) trunk path.
 376     # ancestors[n-1] is the first commit on the new branch.
 377     copyfrom_path = ancestors[len(ancestors)-1]['path']     if ancestors else ""
 378     copyfrom_rev  = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
 379     if ancestors:
 380         # The copy-from path has ancestory back to source_url.
 381         ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
 382             level=ui.DEBUG, color='GREEN', bold=True)
 383         found_ancestor = True
 384         # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
 385         # return None in the case where copyfrom_rev is *before* our source_start_rev.
 386         tgt_rev = get_rev_map(copyfrom_rev, prefix+"  ")
 387         ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
 388     else:
 389         ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
 390         found_ancestor = False
 391     if found_ancestor and tgt_rev:
 392         # Check if this path_offset in the target WC already has this ancestry, in which
 393         # case there's no need to run the "svn copy" (again).
 394         path_in_svn = in_svn(path_offset, prefix=prefix+"  ")
 395         log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+"  ") else []
 396         if (not log_entry or (log_entry['revision'] != tgt_rev)):
 397             copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
 398             ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 399             ui.status(prefix + "   copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
 400             ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
 401             if path_in_svn and \
 402                ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
 403                 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
 404                 # When being called recursively, if this child entry has the same ancestor as the
 405                 # the parent, then no need to try to run another "svn copy".
 406                 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
 407                     parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
 408                 pass
 409             else:
 410                 # Copy this path from the equivalent path+rev in the target repo, to create the
 411                 # equivalent history.
 412                 if parent_copyfrom_path:
 413                     # If we have a parent copy-from path, we mis-match that so display a status
 414                     # message describing the action we're mimic'ing. If path_in_svn, then this
 415                     # is logically a "replace" rather than an "add".
 416                     ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
 417                 if path_in_svn:
 418                     # If local file is already under version-control, then this is a replace.
 419                     ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
 420                     run_svn(["remove", "--force", path_offset])
 421                 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
 422                 if is_dir:
 423                     # Export the final verison of all files in this folder.
 424                     add_path(export_paths, path_offset)
 425                 else:
 426                     # Export the final verison of this file.
 427                     run_svn(["export", "--force", "-r", source_rev,
 428                              source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 429                 # Copy SVN properties from source repo
 430                 sync_svn_props(source_url, source_rev, path_offset)
 431         else:
 432             ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
 433     else:
 434         # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
 435         # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
 436         # Create (parent) directory if needed.
 437         # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
 438         #       split-out to a shared tag?
 439         p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
 440         if not os.path.exists(p_path):
 441             run_svn(["mkdir", p_path])
 442         if not in_svn(path_offset, prefix=prefix+"  "):
 443             if is_dir:
 444                 # Export the final verison of all files in this folder.
 445                 add_path(export_paths, path_offset)
 446             else:
 447                 # Export the final verison of this file. We *need* to do this before running
 448                 # the "svn add", even if we end-up re-exporting this file again via export_paths.
 449                 run_svn(["export", "--force", "-r", source_rev,
 450                          source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
 451             # If not already under version-control, then "svn add" this file/folder.
 452             run_svn(["add", "--parents", path_offset])
 453         # Copy SVN properties from source repo
 454         sync_svn_props(source_url, source_rev, path_offset)
 455     if is_dir:
 456         # For any folders that we process, process any child contents, so that we correctly
 457         # replay copies/replaces/etc.
 458         do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, skip_paths, prefix+"  ")
 459
 460 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
 461                    export_paths, skip_paths, prefix=""):
 462     # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
 463     # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
 464     #       associated remote repo folder. (Is this a problem?)
 465     paths_local =  get_svn_dirlist(path_offset)
 466     paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
 467     ui.status(prefix + ">> do_svn_add_dir: paths_local:  %s", str(paths_local),  level=ui.DEBUG, color='GREEN')
 468     ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
 469     # Update files/folders which exist in remote but not local
 470     for path in paths_remote:
 471         path_is_dir = True if path[-1] == "/" else False
 472         working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
 473         if not working_path in skip_paths:
 474             do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
 475                        export_paths, path_is_dir, skip_paths, prefix+"  ")
 476     # Remove files/folders which exist in local but not remote
 477     for path in paths_local:
 478         if not path in paths_remote:
 479             ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
 480             run_svn(["remove", "--force", path_offset+"/"+path])
 481             # TODO: Does this handle deleted folders too? Wouldn't want to have a case
 482             #       where we only delete all files from folder but leave orphaned folder around.
 483
 484 def process_svn_log_entry(log_entry, commit_paths, prefix = ""):
 485     """
 486     Process SVN changes from the given log entry. Build an array (commit_paths)
 487     of the paths in the working-copy that were changed, i.e. the paths which
 488     we'll pass to "svn commit".
 489     """
 490     export_paths = []
 491     source_rev = log_entry['revision']
 492     ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
 493     for d in log_entry['changed_paths']:
 494         # Get the full path for this changed_path
 495         # e.g. '/branches/bug123/projectA/file1.txt'
 496         path = d['path']
 497         if not is_child_path(path, source_base):
 498             # Ignore changed files that are not part of this subdir
 499             ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s  (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
 500             continue
 501         if d['kind'] == "":
 502             # The "kind" value was introduced in SVN 1.6, and "svn log --xml" won't return a "kind"
 503             # value for commits made on a pre-1.6 repo, even if the server is now running 1.6.
 504             # We need to use other methods to fetch the node-kind for these cases.
 505             d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
 506         assert (d['kind'] == 'file') or (d['kind'] == 'dir')
 507         path_is_dir =  True if d['kind'] == 'dir'  else False
 508         path_is_file = True if d['kind'] == 'file' else False
 509         # Calculate the offset (based on source_base) for this changed_path
 510         # e.g. 'projectA/file1.txt'
 511         # (path = source_base + "/" + path_offset)
 512         path_offset = path[len(source_base):].strip("/")
 513         # Get the action for this path
 514         action = d['action']
 515         if action not in _valid_svn_actions:
 516             raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
 517                 % (source_rev, action))
 518         ui.status(" %s %s%s", action, d['path'],
 519             (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
 520             level=ui.VERBOSE)
 521
 522         # Try to be efficient and keep track of an explicit list of paths in the
 523         # working copy that changed. If we commit from the root of the working copy,
 524         # then SVN needs to crawl the entire working copy looking for pending changes.
 525         commit_paths.append(path_offset)
 526
 527         # Special-handling for replace's
 528         if action == 'R':
 529             # If file was "replaced" (deleted then re-added, all in same revision),
 530             # then we need to run the "svn rm" first, then change action='A'. This
 531             # lets the normal code below handle re-"svn add"'ing the files. This
 532             # should replicate the "replace".
 533             if in_svn(path_offset):
 534                 # Target path might not be under version-control yet, e.g. parent "add"
 535                 # was a copy-from a branch which had no ancestry back to trunk, and each
 536                 # child folder under that parent folder is a "replace" action on the final
 537                 # merge to trunk. Since the child folders will be in skip_paths, do_svn_add
 538                 # wouldn't have created them while processing the parent "add" path.
 539                 run_svn(["remove", "--force", path_offset])
 540             action = 'A'
 541
 542         # Handle all the various action-types
 543         # (Handle "add" first, for "svn copy/move" support)
 544         if action == 'A':
 545             # Determine where to export from.
 546             svn_copy = False
 547             # Handle cases where this "add" was a copy from another URL in the source repo
 548             if d['copyfrom_revision']:
 549                 copyfrom_path = d['copyfrom_path']
 550                 copyfrom_rev =  d['copyfrom_revision']
 551                 skip_paths = []
 552                 for tmp_d in log_entry['changed_paths']:
 553                     tmp_path = tmp_d['path']
 554                     if is_child_path(tmp_path, path):
 555                         # Build list of child entries which are also in the changed_paths list,
 556                         # so that do_svn_add() can skip processing these entries when recursing
 557                         # since we'll end-up processing them later.
 558                         tmp_path_offset = tmp_path[len(source_base):].strip("/")
 559                         skip_paths.append(tmp_path_offset)
 560                 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, skip_paths, prefix+"  ")
 561             # Else just "svn export" the files from the source repo and "svn add" them.
 562             else:
 563                 # Create (parent) directory if needed
 564                 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
 565                 if not os.path.exists(p_path):
 566                     run_svn(["mkdir", p_path])
 567                 # Export the entire added tree.
 568                 if path_is_dir:
 569                     # For directories, defer the (recurisve) "svn export". Might have a
 570                     # situation in a branch merge where the entry in the svn-log is a
 571                     # non-copy-from'd "add" but there are child contents (that we haven't
 572                     # gotten to yet in log_entry) that are copy-from's.  When we try do
 573                     # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
 574                     # having pre-existing (svn-add'd) contents creates some trouble.
 575                     # Instead, just create the stub folders ("svn mkdir" above) and defer
 576                     # exporting the final file-state until the end.
 577                     add_path(export_paths, path_offset)
 578                 else:
 579                     # Export the final verison of this file. We *need* to do this before running
 580                     # the "svn add", even if we end-up re-exporting this file again via export_paths.
 581                     run_svn(["export", "--force", "-r", source_rev,
 582                              source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 583                 if not in_svn(path_offset, prefix=prefix+"  "):
 584                     # Need to use in_svn here to handle cases where client committed the parent
 585                     # folder and each indiv sub-folder.
 586                     run_svn(["add", "--parents", path_offset])
 587                 # Copy SVN properties from source repo
 588                 sync_svn_props(source_url, source_rev, path_offset)
 589
 590         elif action == 'D':
 591             run_svn(["remove", "--force", path_offset])
 592
 593         elif action == 'M':
 594             if path_is_file:
 595                 run_svn(["export", "--force", "-N" , "-r", source_rev,
 596                          source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 597             # Update SVN properties based on source repo
 598             sync_svn_props(source_url, source_rev, path_offset)
 599
 600         else:
 601             raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
 602                 % action)
 603
 604     # Export the final version of all add'd paths from source_url
 605     if export_paths:
 606         for path_offset in export_paths:
 607             run_svn(["export", "--force", "-r", source_rev,
 608                      source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 609
 610 def disp_svn_log_summary(log_entry):
 611     ui.status("")
 612     ui.status("r%s | %s | %s",
 613         log_entry['revision'],
 614         log_entry['author'],
 615         str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
 616     ui.status(log_entry['message'])
 617     ui.status("------------------------------------------------------------------------")
 618
 619 def real_main(args, parser):
 620     global source_url, target_url, rev_map
 621     source_url = args.pop(0).rstrip("/")    # e.g. 'http://server/svn/source/trunk'
 622     target_url = args.pop(0).rstrip("/")    # e.g. 'file:///svn/target/trunk'
 623     ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
 624
 625     # Make sure that both the source and target URL's are valid
 626     source_info = svnclient.get_svn_info(source_url)
 627     assert is_child_path(source_url, source_info['repos_url'])
 628     target_info = svnclient.get_svn_info(target_url)
 629     assert is_child_path(target_url, target_info['repos_url'])
 630
 631     # Init global vars
 632     global source_repos_url,source_base,source_repos_uuid
 633     source_repos_url = source_info['repos_url']       # e.g. 'http://server/svn/source'
 634     source_base = source_url[len(source_repos_url):]  # e.g. '/trunk'
 635     source_repos_uuid = source_info['repos_uuid']
 636
 637     # Init start and end revision
 638     try:
 639         source_start_rev = svnclient.get_svn_rev(source_repos_url, options.rev_start if options.rev_start else 1)
 640     except ExternalCommandFailed:
 641         parser.error("invalid start source revision value: %s" % (options.rev_start))
 642     try:
 643         source_end_rev   = svnclient.get_svn_rev(source_repos_url, options.rev_end   if options.rev_end   else "HEAD")
 644     except ExternalCommandFailed:
 645         parser.error("invalid end source revision value: %s" % (options.rev_end))
 646     ui.status("Using source revision range %s:%s", source_start_rev, source_end_rev, level=ui.VERBOSE)
 647
 648     target_end_rev = target_info['revision']   # Last revision # in the target repo
 649     wc_target = os.path.abspath('_wc_target')
 650     num_entries_proc = 0
 651     commit_count = 0
 652     source_rev = None
 653     target_rev = None
 654
 655     # Check out a working copy of target_url if needed
 656     wc_exists = os.path.exists(wc_target)
 657     if wc_exists and not options.cont_from_break:
 658         shutil.rmtree(wc_target)
 659         wc_exists = False
 660     if not wc_exists:
 661         ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
 662         svnclient.svn_checkout(target_url, wc_target)
 663     os.chdir(wc_target)
 664
 665     if not options.cont_from_break:
 666         # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
 667         # Get the first log entry at/after source_start_rev, which is where
 668         # we'll do the initial import from.
 669         it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False)
 670         for source_start_log in it_log_start:
 671             break
 672         if not source_start_log:
 673             raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
 674                 (source_start_rev, source_end_rev, source_url))
 675
 676         # This is the revision we will start from for source_url
 677         source_start_rev = source_rev = int(source_start_log['revision'])
 678         ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
 679
 680         # For the initial commit to the target URL, export all the contents from
 681         # the source URL at the start-revision.
 682         disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
 683         ui.status("(Initial import)", level=ui.VERBOSE)
 684         # Export and add file-contents from source_url@source_start_rev
 685         top_paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
 686         top_paths = top_paths.strip("\n").split("\n")
 687         for path in top_paths:
 688             # For each top-level file/folder...
 689             if not path:
 690                 continue
 691             # Directories have a trailing slash in the "svn list" output
 692             path_is_dir = True if path[-1] == "/" else False
 693             path_offset = path.rstrip('/') if path_is_dir else path
 694             if in_svn(path_offset, prefix="  "):
 695                 raise InternalError("Cannot replay history on top of pre-existing structure: %s" % source_url+"/"+path_offset)
 696             if path_is_dir and not os.path.exists(path_offset):
 697                 os.makedirs(path_offset)
 698             run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
 699             run_svn(["add", path_offset])
 700         # Update any properties on the newly added content
 701         paths = run_svn(["list", "--recursive", "-r", source_rev, source_url+"@"+str(source_rev)])
 702         paths = paths.strip("\n").split("\n")
 703         sync_svn_props(source_url, source_rev, "")
 704         for path in paths:
 705             if not path:
 706                 continue
 707             # Directories have a trailing slash in the "svn list" output
 708             path_is_dir = True if path[-1] == "/" else False
 709             path_offset = path.rstrip('/') if path_is_dir else path
 710             ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path_offset, level=ui.VERBOSE)
 711             sync_svn_props(source_url, source_rev, path_offset)
 712         # Commit the initial import
 713         num_entries_proc += 1
 714         target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 715         target_rev = commit_from_svn_log_entry(source_start_log, target_revprops=target_revprops)
 716         if target_rev:
 717             # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 718             set_rev_map(source_rev, target_rev)
 719             # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 720             run_svn(["update"])
 721             commit_count += 1
 722     else:
 723         # Re-build the rev_map based on any already-replayed history in target_url
 724         build_rev_map(target_url, target_end_rev, source_info)
 725         if not rev_map:
 726             raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url)
 727         source_start_rev = int(max(rev_map, key=rev_map.get))
 728         assert source_start_rev
 729         ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
 730
 731     svn_vers_t = svnclient.get_svn_client_version()
 732     svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
 733
 734     # Load SVN log starting from source_start_rev + 1
 735     it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else []
 736     source_rev = None
 737
 738     try:
 739         for log_entry in it_log_entries:
 740             if options.entries_proc_limit:
 741                 if num_entries_proc >= options.entries_proc_limit:
 742                     break
 743             # Replay this revision from source_url into target_url
 744             disp_svn_log_summary(log_entry)
 745             source_rev = log_entry['revision']
 746             # Process all the changed-paths in this log entry
 747             commit_paths = []
 748             process_svn_log_entry(log_entry, commit_paths)
 749             num_entries_proc += 1
 750             # Commit any changes made to _wc_target
 751             target_revprops = gen_tracking_revprops(source_rev)   # Build source-tracking revprop's
 752             target_rev = commit_from_svn_log_entry(log_entry, commit_paths, target_revprops=target_revprops)
 753             if target_rev:
 754                 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
 755                 source_rev = log_entry['revision']
 756                 set_rev_map(source_rev, target_rev)
 757                 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
 758                 run_svn(["update"])
 759                 commit_count += 1
 760                 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
 761                 if svn_vers >= 1.7 and (commit_count % 100 == 0):
 762                     run_svn(["cleanup"])
 763         if not source_rev:
 764             # If there were no new source_url revisions to process, init source_rev
 765             # for the "finally" message below.
 766             source_rev = source_end_rev
 767
 768     except KeyboardInterrupt:
 769         print "\nStopped by user."
 770         print "\nCleaning-up..."
 771         run_svn(["cleanup"])
 772         full_svn_revert()
 773     except:
 774         print "\nCommand failed with following error:\n"
 775         traceback.print_exc()
 776         print "\nCleaning-up..."
 777         run_svn(["cleanup"])
 778         print run_svn(["status"])
 779         full_svn_revert()
 780     finally:
 781         print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
 782
 783 def main():
 784     # Defined as entry point. Must be callable without arguments.
 785     usage = "Usage: %prog [OPTIONS] source_url target_url"
 786     description = """\
 787   Replicate (replay) history from one SVN repository to another. Maintain
 788   logical ancestry wherever possible, so that 'svn log' on the replayed
 789   repo will correctly follow file/folder renames.
 790
 791   == Examples ==
 792   Create a copy of only /trunk from source repo, starting at r5000
 793   $ svnadmin create /svn/target
 794   $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
 795   $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
 796     1. The target_url will be checked-out to ./_wc_target
 797     2. The first commit to http://server/source/trunk at/after r5000 will be
 798        exported & added into _wc_target
 799     3. All revisions affecting http://server/source/trunk (starting at r5000)
 800        will be replayed to _wc_target. Any add/copy/move/replaces that are
 801        copy-from'd some path outside of /trunk (e.g. files renamed on a
 802        /branch and branch was merged into /trunk) will correctly maintain
 803        logical ancestry where possible.
 804
 805   Use continue-mode (-c) to pick-up where the last run left-off
 806   $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
 807     1. The target_url will be checked-out to ./_wc_target, if not already
 808        checked-out
 809     2. All new revisions affecting http://server/source/trunk starting from
 810        the last replayed revision to file:///svn/target/trunk (based on the
 811        svn2svn:* revprops) will be replayed to _wc_target, maintaining all
 812        logical ancestry where possible."""
 813     parser = optparse.OptionParser(usage, description=description,
 814                 formatter=HelpFormatter(), version="%prog "+str(full_version))
 815     #parser.remove_option("--help")
 816     #parser.add_option("-h", "--help", dest="show_help", action="store_true",
 817     #    help="show this help message and exit")
 818     parser.add_option("-r", "--revision", type="string", dest="svn_rev", metavar="ARG",
 819                       help="revision range to replay from source_url\n" + \
 820                            "A revision argument can be one of:\n" + \
 821                            "   START        start rev # (end will be 'HEAD')\n" + \
 822                            "   START:END    start and ending rev #'s\n" + \
 823                            "(Any revision # formats which SVN understands\n" + \
 824                            " are supported, e.g. 'HEAD', '{2010-01-31}', etc.)")
 825     parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", default=False,
 826                       help="maintain original 'Author' info from source repo")
 827     parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
 828                       help="continue from previous break")
 829     parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
 830                       help="maximum number of log entries to process")
 831     parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
 832                       help="try processing next log entry but don't commit changes to "
 833                            "target working-copy (forces --limit=1)")
 834     parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
 835                       help="enable additional output (use -vv or -vvv for more)")
 836     parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
 837                       help="enable debugging output (same as -vvv)")
 838     global options
 839     options, args = parser.parse_args()
 840     if len(args) != 2:
 841         parser.error("incorrect number of arguments")
 842     if options.verbosity < 10:
 843         # Expand multiple "-v" arguments to a real ui._level value
 844         options.verbosity *= 10
 845     if options.dry_run:
 846         # When in dry-run mode, only try to process the next log_entry
 847         options.entries_proc_limit = 1
 848     options.rev_start = None
 849     options.rev_end   = None
 850     if options.revision:
 851         # Reg-ex for matching a revision arg (http://svnbook.red-bean.com/en/1.5/svn.tour.revs.specifiers.html#svn.tour.revs.dates)
 852         rev_patt = '[0-9A-Z]+|\{[0-9A-Za-z/\\ :-]+\}'
 853         rev = None
 854         match = re.match('^('+rev_patt+'):('+rev_patt+')$', options.revision)  # First try start:end match
 855         if match is None: match = re.match('^('+rev_patt+')$', options.revision)   # Next, try start match
 856         if match is None:
 857             parser.error("unexpected --revision argument format; see 'svn help log' for valid revision formats")
 858         rev = match.groups()
 859         options.rev_start = rev[0] if len(rev)>0 else None
 860         options.rev_end   = rev[1] if len(rev)>1 else None
 861     ui.update_config(options)
 862     return real_main(args, parser)
 863
 864
 865 if __name__ == "__main__":
 866     sys.exit(main() or 0)