""" Replicate (replay) changesets from one SVN repository to another: * Maintains full logical history (e.g. uses "svn copy" for renames). * Maintains original commit messages. * Optionally maintain source author info. (Only supported if accessing target SVN repo via file://) * Cannot maintain original commit date, but appends original commit date for each commit message: "Date: %d". * Optionally run an external shell script before each replayed commit to give the ability to dynamically exclude or modify files as part of the replay. License: GPLv3, same as hgsvn (https://bitbucket.org/andialbrecht/hgsvn) Author: Tony Duckles (https://github.com/tonyduckles/svn2svn) (Inspired by http://code.google.com/p/svn2svn/, and uses code for hgsvn for SVN client handling) """ from .. import base_version, full_version from .. import ui from .. import svnclient from ..shell import run_svn from ..errors import (ExternalCommandFailed, UnsupportedSVNAction) import sys import os import time import traceback import shutil import operator from optparse import OptionParser,OptionGroup from datetime import datetime def commit_from_svn_log_entry(entry, files=None, keep_author=False, source_props=[]): """ Given an SVN log entry and an optional sequence of files, do an svn commit. """ # TODO: Run optional external shell hook here, for doing pre-commit filtering # This will use the local timezone for displaying commit times timestamp = int(entry['date']) svn_date = str(datetime.fromtimestamp(timestamp)) # Uncomment this one one if you prefer UTC commit times #svn_date = "%d 0" % timestamp if keep_author: options = ["commit", "--force-log", "-m", entry['message'] + "\nDate: " + svn_date, "--username", entry['author']] else: options = ["commit", "--force-log", "-m", entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + entry['author']] if source_props: revprops = [{'name':'svn2svn:source_uuid', 'value':source_props[0]}, {'name':'svn2svn:source_url', 'value':source_props[1]}, {'name':'svn2svn:source_rev', 'value':source_props[2]}] for r in revprops: options += ["--with-revprop", r['name']+"="+str(r['value'])] if files: options += list(files) output = run_svn(options) rev = None if output: output_lines = output.strip("\n").split("\n") rev = "" for line in output_lines: if line[0:19] == 'Committed revision ': rev = line[19:].rstrip('.') break if rev: ui.status("Committed revision %s.", rev) return rev def full_svn_revert(): """ Do an "svn revert" and proactively remove any extra files in the working copy. """ run_svn(["revert", "--recursive", "."]) output = run_svn(["status"]) if output: output_lines = output.strip("\n").split("\n") for line in output_lines: if line[0] == "?": path = line[4:].strip(" ") if os.path.isfile(path): os.remove(path) if os.path.isdir(path): shutil.rmtree(path) def in_svn(p, require_in_repo=False, prefix=""): """ Check if a given file/folder is being tracked by Subversion. Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories. With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy. Use "svn status" to check the status of the file/folder. """ entries = svnclient.get_svn_status(p, no_recursive=True) if not entries: return False d = entries[0] if require_in_repo and (d['status'] == 'added' or d['revision'] is None): # If caller requires this path to be in the SVN repo, prevent returning True # for paths that are only locally-added. ret = False else: # Don't consider files tracked as deleted in the WC as under source-control. # Consider files which are locally added/copied as under source-control. ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN') return ret def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""): """ Given a source path, walk the SVN history backwards to inspect the ancestory of that path, seeing if it traces back to base_path. Build an array of copyfrom_path and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path which base_path is a substring match of (e.g. we crawled back to the initial branch- copy from trunk), then return the collection of ancestor paths. Otherwise, copyfrom_path has no ancestory compared to base_path. This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a file/folder was renamed in a branch and then that branch was merged back to trunk. 'svn_repos_url' is the full URL to the root of the SVN repository, e.g. 'file:///path/to/repo' 'base_path' is the path in the SVN repo to the target path we're trying to trace ancestry back to, e.g. 'trunk'. 'source_path' is the path in the SVN repo to the source path to start checking ancestry at, e.g. 'branches/fix1/projectA/file1.txt'. (full_path = svn_repos_url+base_path+"/"+path_offset) 'source_rev' is the revision to start walking the history of source_path backwards from. """ ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s base_path: %s", svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW') done = False working_path = base_path+"/"+source_path working_rev = source_rev first_iter_done = False ancestors_temp = [] while not done: # Get the first "svn log" entry for this path (relative to @rev) ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW') log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path+"@"+str(working_rev), 1, working_rev, True) if not log_entry: ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW') done = True break # If we found a copy-from case which matches our base_path, we're done. # ...but only if we've at least tried to search for the first copy-from path. if first_iter_done and working_path.startswith(base_path): ui.status(prefix + ">> find_svn_ancestors: Done: Found working_path.startswith(base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW') done = True break first_iter_done = True # Search for any actions on our target path (or parent paths). changed_paths_temp = [] for d in log_entry['changed_paths']: path = d['path'] if path in working_path: changed_paths_temp.append({'path': path, 'data': d}) if not changed_paths_temp: # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path. ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW') done = True continue # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path. changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True) # Find the action for our working_path in this revision. Use a loop to check in reverse order, # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from. for v in changed_paths: d = v['data'] path = d['path'] # Check action-type for this file action = d['action'] if action not in 'MARD': raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!" % (log_entry['revision'], action)) ui.status(prefix + "> %s %s%s", action, path, (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "", level=ui.DEBUG, color='YELLOW') if action == 'D': # If file/folder was deleted, it has no ancestor ancestors_temp = [] ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW') done = True break if action in 'RA': # If file/folder was added/replaced but not a copy, it has no ancestor if not d['copyfrom_path']: ancestors_temp = [] ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path", "Added" if action == "A" else "Replaced", level=ui.DEBUG, color='YELLOW') done = True break # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list # and keep checking for ancestors ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s", action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']), level=ui.DEBUG, color='YELLOW') ancestors_temp.append({'path': path, 'revision': log_entry['revision'], 'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']}) working_path = working_path.replace(d['path'], d['copyfrom_path']) working_rev = d['copyfrom_revision'] # Follow the copy and keep on searching break ancestors = [] if ancestors_temp: ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev}) working_path = base_path+"/"+source_path for idx in range(len(ancestors_temp)): d = ancestors_temp[idx] working_path = working_path.replace(d['path'], d['copyfrom_path']) working_rev = d['copyfrom_rev'] ancestors.append({'path': working_path, 'revision': working_rev}) max_len = 0 for idx in range(len(ancestors)): d = ancestors[idx] max_len = max(max_len, len(d['path']+"@"+str(d['revision']))) ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B') for idx in range(len(ancestors)-1): d = ancestors[idx] d_next = ancestors[idx+1] ui.status(prefix + " [%s] %s <-- %s", idx, str(d['path']+"@"+str(d['revision'])).ljust(max_len), str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len), level=ui.DEBUG, color='YELLOW') else: ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s", svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW') return ancestors def get_rev_map(rev_map, source_rev, prefix): """ Find the equivalent rev # in the target repo for the given rev # from the source repo. """ ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN') # Find the highest entry less-than-or-equal-to source_rev for rev in range(int(source_rev), 0, -1): ui.status(prefix + ">> get_rev_map: rev=%s in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B') if rev in rev_map: return int(rev_map[rev]) # Else, we fell off the bottom of the rev_map. Ruh-roh... return None def set_rev_map(rev_map, source_rev, target_rev): ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN') rev_map[int(source_rev)]=int(target_rev) def build_rev_map(target_url, source_info): """ Check for any already-replayed history from source_url (source_info) and build the mapping-table of source_rev -> target_rev. """ rev_map = {} ui.status("Rebuilding rev_map...", level=ui.VERBOSE) proc_count = 0 it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, 'HEAD', get_changed_paths=False, get_revprops=True) for log_entry in it_log_entries: if log_entry['revprops']: revprops = {} for v in log_entry['revprops']: if v['name'].startswith('svn2svn:'): revprops[v['name']] = v['value'] if revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \ revprops['svn2svn:source_url'] == source_info['url']: source_rev = revprops['svn2svn:source_rev'] target_rev = log_entry['revision'] set_rev_map(rev_map, source_rev, target_rev) return rev_map def get_svn_dirlist(svn_path, svn_rev = ""): """ Get a list of all the child contents (recusive) of the given folder path. """ args = ["list"] path = svn_path if svn_rev: args += ["-r", svn_rev] path += "@"+str(svn_rev) args += [path] paths = run_svn(args, no_fail=True) paths = paths.strip("\n").split("\n") if len(paths)>1 else [] return paths def _add_export_path(export_paths, path_offset): found = False for p in export_paths: if path_offset.startswith(p): found = True break if not found: export_paths.append(path_offset) return export_paths def do_svn_add(source_repos_url, source_url, path_offset, target_url, source_rev, \ parent_copyfrom_path="", parent_copyfrom_rev="", export_paths={}, \ rev_map={}, is_dir = False, prefix = ""): """ Given the add'd source path, replay the "svn add/copy" commands to correctly track renames across copy-from's. For example, consider a sequence of events like this: 1. svn copy /trunk /branches/fix1 2. (Make some changes on /branches/fix1) 3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2 " Rename folder 4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt " Rename file inside renamed folder 5. svn co /trunk && svn merge /branches/fix1 After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1 and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder, we'd lose the logical history that Proj2/file2.txt is really a descendant of Proj1/file1.txt. 'source_repos_url' is the full URL to the root of the source repository. 'source_url' is the full URL to the source path in the source repository. 'path_offset' is the offset from source_base to the file to check ancestry for, e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset. 'target_url' is the full URL to the target path in the target repository. 'source_rev' is the revision ("svn log") that we're processing from the source repo. 'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent directory, when being called recursively by do_svn_add_dir(). 'export_paths' is the list of path_offset's that we've deferred running "svn export" on. 'rev_map' is the running mapping-table dictionary for source-repo rev #'s to the equivalent target-repo rev #'s. 'is_dir' is whether path_offset is a directory (rather than a file). """ source_base = source_url[len(source_repos_url):] ui.status(prefix + ">> do_svn_add: %s %s", source_base+"/"+path_offset+"@"+str(source_rev), " (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN') # Check if the given path has ancestors which chain back to the current source_base found_ancestor = False ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+" ") # ancestors[n] is the original (pre-branch-copy) trunk path. # ancestors[n-1] is the first commit on the new branch. copyfrom_path = ancestors[len(ancestors)-1]['path'] if ancestors else "" copyfrom_rev = ancestors[len(ancestors)-1]['revision'] if ancestors else "" if ancestors: # The copy-from path has ancestory back to source_url. ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN', bold=True) found_ancestor = True # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can # return None in the case where copyfrom_rev is *before* our source_start_rev. tgt_rev = get_rev_map(rev_map, copyfrom_rev, prefix+" ") ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN') else: ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN') found_ancestor = False if found_ancestor and tgt_rev: # Check if this path_offset in the target WC already has this ancestry, in which # case there's no need to run the "svn copy" (again). path_in_svn = in_svn(path_offset, prefix=prefix+" ") log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+" ") else [] if (not log_entry or (log_entry['revision'] != tgt_rev)): copyfrom_offset = copyfrom_path[len(source_base):].strip('/') ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN') ui.status(prefix + " copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN') ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN') if path_in_svn and \ ((parent_copyfrom_path and copyfrom_path.startswith(parent_copyfrom_path)) and \ (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)): # When being called recursively, if this child entry has the same ancestor as the # the parent, then no need to try to run another "svn copy". ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN') pass else: # Copy this path from the equivalent path+rev in the target repo, to create the # equivalent history. if parent_copyfrom_path: # If we have a parent copy-from path, we mis-match that so display a status # message describing the action we're mimic'ing. If path_in_svn, then this # is logically a "replace" rather than an "add". ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE) if path_in_svn: # If local file is already under version-control, then this is a replace. ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN') run_svn(["remove", "--force", path_offset]) run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset]) # Export the final version of this file/folder from the source repo, to make # sure we're up-to-date. export_paths = _add_export_path(export_paths, path_offset) else: ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN') else: # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy". # Create (parent) directory if needed. # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be # split-out to a shared tag? p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.' if not os.path.exists(p_path): run_svn(["mkdir", p_path]) if not in_svn(path_offset, prefix=prefix+" "): if is_dir: # Export the final verison of all files in this folder. export_paths = _add_export_path(export_paths, path_offset) else: # Export the final verison of this file. We *need* to do this before running # the "svn add", even if we end-up re-exporting this file again via export_paths. run_svn(["export", "--force", "-r", source_rev, source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset]) # If not already under version-control, then "svn add" this file/folder. run_svn(["add", "--parents", path_offset]) # TODO: Need to copy SVN properties from source repos if is_dir: # For any folders that we process, process any child contents, so that we correctly # replay copies/replaces/etc. do_svn_add_dir(source_repos_url, source_url, path_offset, source_rev, target_url, copyfrom_path, copyfrom_rev, export_paths, rev_map, prefix+" ") def do_svn_add_dir(source_repos_url, source_url, path_offset, source_rev, target_url, \ parent_copyfrom_path, parent_copyfrom_rev, export_paths, rev_map, prefix=""): source_base = source_url[len(source_repos_url):] # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url) # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the # associated remote repo folder. (Is this a problem?) paths_local = get_svn_dirlist(path_offset) paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev) ui.status(prefix + ">> do_svn_add_dir: paths_local: %s", str(paths_local), level=ui.DEBUG, color='GREEN') ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN') # Update files/folders which exist in remote but not local for path in paths_remote: path_is_dir = True if path[-1] == "/" else False working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path) do_svn_add(source_repos_url, source_url, working_path, target_url, source_rev, parent_copyfrom_path, parent_copyfrom_rev, export_paths, rev_map, path_is_dir, prefix+" ") # Remove files/folders which exist in local but not remote for path in paths_local: if not path in paths_remote: ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE) run_svn(["remove", "--force", path_offset+"/"+path]) # TODO: Does this handle deleted folders too? Wouldn't want to have a case # where we only delete all files from folder but leave orphaned folder around. def process_svn_log_entry(log_entry, source_repos_url, source_url, target_url, \ rev_map, commit_paths = [], prefix = ""): """ Process SVN changes from the given log entry. Returns array of all the paths in the working-copy that were changed, i.e. the paths which need to be "svn commit". 'log_entry' is the array structure built by parse_svn_log_xml(). 'source_repos_url' is the full URL to the root of the source repository. 'source_url' is the full URL to the source path in the source repository. 'target_url' is the full URL to the target path in the target repository. 'rev_map' is the running mapping-table dictionary for source-repo rev #'s to the equivalent target-repo rev #'s. 'commit_paths' is the working list of specific paths which changes to pass to the final "svn commit". """ removed_paths = [] export_paths = [] # Get the relative offset of source_url based on source_repos_url # e.g. '/branches/bug123' source_base = source_url[len(source_repos_url):] source_rev = log_entry['revision'] ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN') for d in log_entry['changed_paths']: # Get the full path for this changed_path # e.g. '/branches/bug123/projectA/file1.txt' path = d['path'] if not path.startswith(source_base + "/"): # Ignore changed files that are not part of this subdir if path != source_base: ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN') continue # Calculate the offset (based on source_base) for this changed_path # e.g. 'projectA/file1.txt' # (path = source_base + "/" + path_offset) path_offset = path[len(source_base):].strip("/") # Get the action for this path action = d['action'] if action not in 'MARD': raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!" % (source_rev, action)) if action not in 'D': # (Note: Skip displaying action message for 'D' here since we'll display that # message when we process the deferred delete actions at the end.) ui.status(" %s %s%s", action, d['path'], (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "", level=ui.VERBOSE) # Try to be efficient and keep track of an explicit list of paths in the # working copy that changed. If we commit from the root of the working copy, # then SVN needs to crawl the entire working copy looking for pending changes. # But, if we gather too many paths to commit, then we wipe commit_paths below # and end-up doing a commit at the root of the working-copy. if len (commit_paths) < 100: commit_paths.append(path_offset) # Special-handling for replace's if action == 'R': # If file was "replaced" (deleted then re-added, all in same revision), # then we need to run the "svn rm" first, then change action='A'. This # lets the normal code below handle re-"svn add"'ing the files. This # should replicate the "replace". run_svn(["remove", "--force", path_offset]) action = 'A' # Handle all the various action-types # (Handle "add" first, for "svn copy/move" support) if action == 'A': # If we have any queued deletions for this same path, remove those if we're re-adding this path. if path_offset in removed_paths: removed_paths.remove(path_offset) # Determine where to export from. svn_copy = False path_is_dir = True if d['kind'] == 'dir' else False # Handle cases where this "add" was a copy from another URL in the source repos if d['copyfrom_revision']: copyfrom_path = d['copyfrom_path'] copyfrom_rev = d['copyfrom_revision'] do_svn_add(source_repos_url, source_url, path_offset, target_url, source_rev, "", "", export_paths, rev_map, path_is_dir, prefix+" ") # Else just "svn export" the files from the source repo and "svn add" them. else: # Create (parent) directory if needed p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.' if not os.path.exists(p_path): run_svn(["mkdir", p_path]) # Export the entire added tree. if path_is_dir: export_paths = _add_export_path(export_paths, path_offset) else: # Export the final verison of this file. We *need* to do this before running # the "svn add", even if we end-up re-exporting this file again via export_paths. run_svn(["export", "--force", "-r", source_rev, source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset]) if not in_svn(path_offset, prefix=prefix+" "): # Need to use in_svn here to handle cases where client committed the parent # folder and each indiv sub-folder. run_svn(["add", "--parents", path_offset]) # TODO: Need to copy SVN properties from source repos elif action == 'D': # Queue "svn remove" commands, to allow the action == 'A' handling the opportunity # to do smart "svn copy" handling on copy/move/renames. if not path_offset in removed_paths: removed_paths.append(path_offset) elif action == 'M': # TODO: Is "svn merge -c" correct here? Should this just be an "svn export" plus # proplist updating? out = run_svn(["merge", "-c", source_rev, "--non-recursive", "--non-interactive", "--accept=theirs-full", source_url+"/"+path_offset+"@"+str(source_rev), path_offset]) else: raise SVNError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'" % action) # Process any deferred removed actions if removed_paths: path_base = source_url[len(source_repos_url):] for path_offset in removed_paths: ui.status(" %s %s", 'D', path_base+"/"+path_offset, level=ui.VERBOSE) run_svn(["remove", "--force", path_offset]) # Export the final version of all add'd paths from source_url if export_paths: for path_offset in export_paths: run_svn(["export", "--force", "-r", source_rev, source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset]) return commit_paths def disp_svn_log_summary(log_entry): ui.status("") ui.status("r%s | %s | %s", log_entry['revision'], log_entry['author'], str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' '))) ui.status(log_entry['message']) ui.status("------------------------------------------------------------------------") def pull_svn_rev(log_entry, source_repos_url, source_repos_uuid, source_url, target_url, rev_map, keep_author=False): """ Pull SVN changes from the given log entry. Returns the new SVN revision. If an exception occurs, it will rollback to revision 'source_rev - 1'. """ disp_svn_log_summary(log_entry) source_rev = log_entry['revision'] # Process all the paths in this log entry commit_paths = [] process_svn_log_entry(log_entry, source_repos_url, source_url, target_url, rev_map, commit_paths) # If we had too many individual paths to commit, wipe the list and just commit at # the root of the working copy. if len (commit_paths) > 99: commit_paths = [] # Add source-tracking revprop's source_props = [source_repos_uuid, source_url, source_rev] return commit_from_svn_log_entry(log_entry, commit_paths, keep_author=keep_author, source_props=source_props) def run_parser(parser): """ Add common options to an OptionParser instance, and run parsing. """ parser.add_option("", "--version", dest="show_version", action="store_true", help="show version and exit") parser.remove_option("--help") parser.add_option("-h", "--help", dest="show_help", action="store_true", help="show this help message and exit") parser.add_option("-v", "--verbose", dest="verbosity", const=ui.VERBOSE, default=10, action="store_const", help="enable additional output") parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const", help="enable debugging output") options, args = parser.parse_args() if options.show_help: parser.print_help() sys.exit(0) if options.show_version: prog_name = os.path.basename(sys.argv[0]) print prog_name, full_version sys.exit(0) ui.update_config(options) return options, args def display_parser_error(parser, message): """ Display an options error, and terminate. """ print "error:", message print parser.print_help() sys.exit(1) def real_main(options, args): source_url = args.pop(0).rstrip("/") target_url = args.pop(0).rstrip("/") if options.keep_author: keep_author = True else: keep_author = False # Make sure that both the source and target URL's are valid source_info = svnclient.get_svn_info(source_url) assert source_url.startswith(source_info['repos_url']) target_info = svnclient.get_svn_info(target_url) assert target_url.startswith(target_info['repos_url']) source_end_rev = source_info['revision'] # Get the last revision # for the source repo source_repos_url = source_info['repos_url'] # Get the base URL for the source repo, e.g. 'svn://svn.example.com/svn/repo' source_repos_uuid = source_info['repos_uuid'] # Get the UUID for the source repo wc_target = os.path.abspath('_wc_target') rev_map = {} # Check out a working copy of target_url if needed wc_exists = os.path.exists(wc_target) if wc_exists and not options.cont_from_break: shutil.rmtree(wc_target) wc_exists = False if not wc_exists: svnclient.svn_checkout(target_url, wc_target) os.chdir(wc_target) if not options.cont_from_break: # Get log entry for the SVN revision we will check out if options.svn_rev: # If specify a rev, get log entry just before or at rev source_start_log = svnclient.get_last_svn_log_entry(source_url, 1, options.svn_rev, False) else: # Otherwise, get log entry of branch creation # Note: Trying to use svnclient.get_first_svn_log_entry(source_url, 1, source_end_rev, False) # ends-up being *VERY* time-consuming on a repo with lots of revisions. Even though # the "svn log" call is passing --limit 1, it seems like that limit-filter is happening # _after_ svn has fetched the full log history. Instead, search the history in chunks # and write some progress to the screen. ui.status("Searching for start source revision (%s)...", source_url, level=ui.VERBOSE) rev = 1 chunk_size = 1000 done = False while not done: entries = svnclient.run_svn_log(source_url, rev, min(rev+chunk_size-1, target_info['revision']), 1, get_changed_paths=False) if entries: source_start_log = entries[0] done = True break ui.status("...%s...", rev) rev = rev+chunk_size if rev > target_info['revision']: done = True if not source_start_log: raise RuntimeError("Unable to find first revision for source_url: %s" % source_url) # This is the revision we will start from for source_url source_start_rev = source_rev = int(source_start_log['revision']) ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE) # For the initial commit to the target URL, export all the contents from # the source URL at the start-revision. paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)]) if len(paths)>1: disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev)) ui.status("(Initial import)", level=ui.VERBOSE) paths = paths.strip("\n").split("\n") for path_raw in paths: # For each top-level file/folder... if not path_raw: continue # Directories have a trailing slash in the "svn list" output path_is_dir = True if path_raw[-1] == "/" else False path = path_raw.rstrip('/') if path_is_dir else path_raw if path_is_dir and not os.path.exists(path): os.makedirs(path) ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path, level=ui.VERBOSE) run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path+"@"+str(source_rev), path]) run_svn(["add", path]) source_props = [source_repos_uuid, source_url, source_rev] target_rev = commit_from_svn_log_entry(source_start_log, [], keep_author=keep_author, source_props=source_props) if target_rev: set_rev_map(rev_map, source_rev, target_rev) else: # Re-build the rev_map based on any already-replayed history in target_url rev_map = build_rev_map(target_url, source_info) if not rev_map: raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url) source_start_rev = int(max(rev_map, key=rev_map.get)) assert source_start_rev ui.status("Continue from source revision %s.", source_start_rev, level=ui.VERBOSE) commit_count = 0 svn_vers_t = svnclient.get_svn_client_version() svn_vers = float(".".join(map(str, svn_vers_t[0:2]))) # Load SVN log starting from source_start_rev + 1 it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev) try: for log_entry in it_log_entries: # Replay this revision from source_url into target_url target_rev = pull_svn_rev(log_entry, source_repos_url, source_repos_uuid, source_url, target_url, rev_map, keep_author) # Update our target working-copy, to ensure everything says it's at the new HEAD revision run_svn(["update"]) commit_count += 1 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*" if svn_vers >= 1.7 and (commit_count % 100 == 0): run_svn(["cleanup"]) # Update rev_map, mapping table of source-repo rev # -> target-repo rev # if target_rev: source_rev = log_entry['revision'] set_rev_map(rev_map, source_rev, target_rev) except KeyboardInterrupt: print "\nStopped by user." run_svn(["cleanup"]) full_svn_revert() except: print "\nCommand failed with following error:\n" traceback.print_exc() run_svn(["cleanup"]) print run_svn(["status"]) full_svn_revert() finally: run_svn(["update"]) print "\nFinished!" def main(): # Defined as entry point. Must be callable without arguments. usage = "Usage: %prog [OPTIONS] source_url target_url" parser = OptionParser(usage) parser.add_option("-r", "--revision", type="int", dest="svn_rev", metavar="REV", help="initial SVN revision to checkout from") parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", help="maintain original Author info from source repo") parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break", help="continue from previous break") (options, args) = run_parser(parser) if len(args) != 2: display_parser_error(parser, "incorrect number of arguments") return real_main(options, args) if __name__ == "__main__": sys.exit(main() or 0)