]> Tony Duckles's Git Repositories (git.nynim.org) - svn2svn.git/blob - svn2svn/run/svn2svn.py
Don't allow starting a fresh (non-continue) replay into a non-empty target directory...
[svn2svn.git] / svn2svn / run / svn2svn.py
1 """
2 Replicate (replay) changesets from one SVN repository to another.
3 """
4
5 from .. import base_version, full_version
6 from .. import ui
7 from .. import svnclient
8 from ..shell import run_svn,run_shell_command
9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
10 from parse import HelpFormatter
11 from breakhandler import BreakHandler
12
13 import sys
14 import os
15 import time
16 import traceback
17 import shutil
18 import operator
19 import optparse
20 import re
21 import urllib
22 from datetime import datetime
23
24 _valid_svn_actions = "MARD" # The list of known SVN action abbr's, from "svn log"
25
26 # Module-level variables/parameters
27 source_url = "" # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
28 source_repos_url = "" # URL to root of source SVN repo, e.g. 'http://server/svn/source'
29 source_base = "" # Relative path of source_url in source SVN repo, e.g. '/trunk'
30 source_repos_uuid = "" # UUID of source SVN repo
31 target_url ="" # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
32 target_repos_url = "" # URL to root of target SVN repo, e.g. 'http://server/svn/target'
33 target_base = "" # Relative path of target_url in target SVN repo, e.g. '/trunk'
34 rev_map = {} # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
35 options = None # optparser options
36
37 def parse_svn_commit_rev(output):
38 """
39 Parse the revision number from the output of "svn commit".
40 """
41 output_lines = output.strip("\n").split("\n")
42 rev_num = None
43 for line in output_lines:
44 if line[0:19] == 'Committed revision ':
45 rev_num = line[19:].rstrip('.')
46 break
47 assert rev_num is not None
48 return int(rev_num)
49
50 def commit_from_svn_log_entry(log_entry, commit_paths=None, target_revprops=None):
51 """
52 Given an SVN log entry and an optional list of changed paths, do an svn commit.
53 """
54 # TODO: Run optional external shell hook here, for doing pre-commit filtering
55 # Display the _wc_target "svn status" info if running in -vv (or higher) mode
56 if ui.get_level() >= ui.EXTRA:
57 ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
58 ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
59 # This will use the local timezone for displaying commit times
60 timestamp = int(log_entry['date'])
61 svn_date = str(datetime.fromtimestamp(timestamp))
62 # Uncomment this one one if you prefer UTC commit times
63 #svn_date = "%d 0" % timestamp
64 args = ["commit", "--force-log"]
65 message = log_entry['message']
66 if options.log_date:
67 message += "\nDate: " + svn_date
68 if options.log_author:
69 message += "\nAuthor: " + log_entry['author']
70 args += ["-m", message]
71 revprops = {}
72 if log_entry['revprops']:
73 # Carry forward any revprop's from the source revision
74 for v in log_entry['revprops']:
75 revprops[v['name']] = v['value']
76 if target_revprops:
77 # Add any extra revprop's we want to set for the target repo commits
78 for v in target_revprops:
79 revprops[v['name']] = v['value']
80 if revprops:
81 for key in revprops:
82 args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
83 if commit_paths:
84 if len(commit_paths)<100:
85 # If we don't have an excessive amount of individual changed paths, pass
86 # those to the "svn commit" command. Else, pass nothing so we commit at
87 # the root of the working-copy.
88 args += list(commit_paths)
89 rev_num = None
90 if not options.dry_run:
91 # Use BreakHandler class to temporarily redirect SIGINT handler, so that
92 # "svn commit" + post-commit rev-prop updating is a quasi-atomic unit.
93 # If user presses Ctrl-C during this, wait until after this full action
94 # has finished raising the KeyboardInterrupt exception.
95 bh = BreakHandler()
96 bh.enable()
97 # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
98 output = run_svn(args)
99 rev_num = parse_svn_commit_rev(output) if output else None
100 if rev_num is not None:
101 if options.keep_date:
102 run_svn(["propset", "--revprop", "-r", rev_num, "svn:date", log_entry['date_raw']])
103 if options.keep_author:
104 run_svn(["propset", "--revprop", "-r", rev_num, "svn:author", log_entry['author']])
105 ui.status("Committed revision %s (source r%s).", rev_num, log_entry['revision'])
106 bh.disable()
107 # Check if the user tried to press Ctrl-C
108 if bh.trapped:
109 raise KeyboardInterrupt
110 return rev_num
111
112 def verify_commit(source_rev, target_rev, log_entry=None):
113 """
114 Compare the ancestry/content/properties between source_url vs target_url
115 for a given revision.
116 """
117 # Gather the offsets in the source repo to check
118 check_paths = []
119 remove_paths = []
120 # TODO: Need to make this ancestry aware
121 if options.verify == 1 and log_entry is not None: # Changed only
122 ui.status("Verifying source revision %s (only-changed)...", source_rev, level=ui.VERBOSE)
123 for d in log_entry['changed_paths']:
124 path = d['path']
125 if not is_child_path(path, source_base):
126 continue
127 if d['kind'] == "":
128 d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
129 assert (d['kind'] == 'file') or (d['kind'] == 'dir')
130 path_is_dir = True if d['kind'] == 'dir' else False
131 path_is_file = True if d['kind'] == 'file' else False
132 path_offset = path[len(source_base):].strip("/")
133 if d['action'] == 'D':
134 remove_paths.append(path_offset)
135 elif not path_offset in check_paths:
136 ui.status("verify_commit: path [mode=changed]: kind=%s: %s", d['kind'], path, level=ui.DEBUG, color='YELLOW')
137 if path_is_file:
138 ui.status(" "+"verify_commit [mode=changed]: check_paths.append('%s')", path_offset, level=ui.DEBUG, color='GREEN')
139 check_paths.append(path_offset)
140 if path_is_dir:
141 if not d['action'] in 'AR':
142 continue
143 child_paths = run_svn(["list", "--recursive", "-r", source_rev, source_url.rstrip("/")+"/"+path_offset+"@"+str(source_rev)])
144 child_paths = child_paths.strip("\n").split("\n")
145 for child_path in child_paths:
146 if not child_path:
147 continue
148 # Directories have a trailing slash in the "svn list" output
149 child_path_is_dir = True if child_path[-1] == "/" else False
150 child_path_offset = child_path.rstrip('/') if child_path_is_dir else child_path
151 if not child_path_is_dir:
152 # Only check files
153 working_path = (path_offset+"/" if path_offset else "") + child_path_offset
154 if not working_path in check_paths:
155 ui.status(" "+"verify_commit [mode=changed]: check_paths.append('%s'+'/'+'%s')", path_offset, child_path_offset, level=ui.DEBUG, color='GREEN')
156 check_paths.append(working_path)
157 if options.verify == 2: # All paths
158 ui.status("Verifying source revision %s (all)...", source_rev, level=ui.VERBOSE)
159 child_paths = run_svn(["list", "--recursive", "-r", source_rev, source_url+"@"+str(source_rev)])
160 child_paths = child_paths.strip("\n").split("\n")
161 for child_path in child_paths:
162 if not child_path:
163 continue
164 # Directories have a trailing slash in the "svn list" output
165 child_path_is_dir = True if child_path[-1] == "/" else False
166 child_path_offset = child_path.rstrip('/') if child_path_is_dir else child_path
167 if not child_path_is_dir:
168 # Only check files
169 ui.status("verify_commit [mode=all]: check_paths.append('%s')", child_path_offset, level=ui.DEBUG, color='GREEN')
170 check_paths.append(child_path_offset)
171
172 # If there were any paths deleted in the last revision (options.verify=1 mode),
173 # check that they were correctly deleted.
174 if remove_paths:
175 count_total = len(remove_paths)
176 count = 0
177 for path_offset in remove_paths:
178 count += 1
179 if in_svn(path_offset):
180 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.VERBOSE, color='RED')
181 raise VerificationError("Path removed in source rev r%s, but still exists in target WC: %s" % (source_rev, path_offset))
182 ui.status(" (%s/%s) Verify remove: OK: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.VERBOSE)
183
184 # Compare each of the check_path entries between source vs. target
185 if check_paths:
186 source_rev_first = int(min(rev_map, key=rev_map.get)) or 1 # The first source_rev we replayed into target
187 ui.status("verify_commit: source_rev_first:%s", source_rev_first, level=ui.DEBUG, color='YELLOW')
188 count_total = len(check_paths)
189 count = 0
190 for path_offset in check_paths:
191 count += 1
192 ui.status("verify_commit: path_offset:%s", path_offset, level=ui.DEBUG, color='YELLOW')
193 source_log_entries = svnclient.run_svn_log(source_url.rstrip("/")+"/"+path_offset+"@"+str(source_rev), source_rev, 1, source_rev-source_rev_first+1)
194 target_log_entries = svnclient.run_svn_log(target_url.rstrip("/")+"/"+path_offset+"@"+str(target_rev), target_rev, 1, target_rev)
195 # Build a list of commits in source_log_entries which matches our
196 # target path_offset.
197 working_path = source_base+"/"+path_offset
198 source_revs = []
199 for log_entry in source_log_entries:
200 source_rev_tmp = log_entry['revision']
201 if source_rev_tmp < source_rev_first:
202 # Only process source revisions which have been replayed into target
203 break
204 #ui.status(" [verify_commit] source_rev_tmp:%s, working_path:%s\n%s", source_rev_tmp, working_path, pp.pformat(log_entry), level=ui.DEBUG, color='MAGENTA')
205 changed_paths_temp = []
206 for d in log_entry['changed_paths']:
207 path = d['path']
208 # Match working_path or any parents
209 if is_child_path(working_path, path):
210 ui.status(" verify_commit: changed_path: %s %s@%s (parent:%s)", d['action'], path, source_rev_tmp, working_path, level=ui.DEBUG, color='YELLOW')
211 changed_paths_temp.append({'path': path, 'data': d})
212 assert changed_paths_temp
213 # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
214 changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
215 # Find the action for our working_path in this revision. Use a loop to check in reverse order,
216 # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
217 working_path_next = working_path
218 match_d = {}
219 for v in changed_paths:
220 d = v['data']
221 if not match_d:
222 match_d = d
223 path = d['path']
224 if d['action'] not in _valid_svn_actions:
225 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
226 % (log_entry['revision'], d['action']))
227 if d['action'] in 'AR' and d['copyfrom_revision']:
228 # If we found a copy-from action for a parent path, adjust our
229 # working_path to follow the rename/copy-from, just like find_svn_ancestors().
230 working_path_next = working_path.replace(d['path'], d['copyfrom_path'])
231 match_d = d
232 break
233 if is_child_path(working_path, source_base):
234 # Only add source_rev's where the path changed in this revision was a child
235 # of source_base, so that we silently ignore any history that happened on
236 # non-source_base paths (e.g. ignore branch history if we're only replaying trunk).
237 is_diff = False
238 d = match_d
239 if d['action'] == 'M':
240 # For action="M", we need to throw out cases where the only change was to
241 # a property which we ignore, e.g. "svn:mergeinfo".
242 if d['kind'] == "":
243 d['kind'] = svnclient.get_kind(source_repos_url, working_path, log_entry['revision'], d['action'], log_entry['changed_paths'])
244 assert (d['kind'] == 'file') or (d['kind'] == 'dir')
245 if d['kind'] == 'file':
246 # Check for file-content changes
247 # TODO: This should be made ancestor-aware, since the file won't always be at the same path in rev-1
248 sum1 = run_shell_command("svn cat -r %s '%s' | md5sum" % (source_rev_tmp, source_repos_url+working_path+"@"+str(source_rev_tmp)))
249 sum2 = run_shell_command("svn cat -r %s '%s' | md5sum" % (source_rev_tmp-1, source_repos_url+working_path_next+"@"+str(source_rev_tmp-1)))
250 is_diff = True if sum1 <> sum2 else False
251 if not is_diff:
252 # Check for property changes
253 props1 = svnclient.get_all_props(source_repos_url+working_path, source_rev_tmp)
254 props2 = svnclient.get_all_props(source_repos_url+working_path_next, source_rev_tmp-1)
255 # Ignore changes to "svn:mergeinfo", since we don't copy that
256 if 'svn:mergeinfo' in props1: del props1['svn:mergeinfo']
257 if 'svn:mergeinfo' in props2: del props2['svn:mergeinfo']
258 for prop in props1:
259 if prop not in props2 or \
260 props1[prop] != props2[prop]:
261 is_diff = True
262 break
263 for prop in props2:
264 if prop not in props1 or \
265 props1[prop] != props2[prop]:
266 is_diff = True
267 break
268 if not is_diff:
269 ui.status(" verify_commit: skip %s@%s", working_path, source_rev_tmp, level=ui.DEBUG, color='GREEN_B', bold=True)
270 else:
271 is_diff = True
272 if is_diff:
273 ui.status(" verify_commit: source_revs.append(%s), working_path:%s", source_rev_tmp, working_path, level=ui.DEBUG, color='GREEN_B')
274 source_revs.append({'path': working_path, 'revision': source_rev_tmp})
275 working_path = working_path_next
276 # Build a list of all the target commits "svn log" returned
277 target_revs = []
278 target_revs_rmndr = []
279 for log_entry in target_log_entries:
280 target_rev_tmp = log_entry['revision']
281 ui.status(" verify_commit: target_revs.append(%s)", target_rev_tmp, level=ui.DEBUG, color='GREEN_B')
282 target_revs.append(target_rev_tmp)
283 target_revs_rmndr.append(target_rev_tmp)
284 # Compare the two lists
285 for d in source_revs:
286 working_path = d['path']
287 source_rev_tmp = d['revision']
288 target_rev_tmp = get_rev_map(source_rev_tmp, " ")
289 working_offset = working_path[len(source_base):].strip("/")
290 sum1 = run_shell_command("svn cat -r %s '%s' | md5sum" % (source_rev_tmp, source_repos_url+working_path+"@"+str(source_rev_tmp)))
291 sum2 = run_shell_command("svn cat -r %s '%s' | md5sum" % (target_rev_tmp, target_url+"/"+working_offset+"@"+str(target_rev_tmp))) if target_rev_tmp is not None else ""
292 #print "source@%s: %s" % (str(source_rev_tmp).ljust(6), sum1)
293 #print "target@%s: %s" % (str(target_rev_tmp).ljust(6), sum2)
294 ui.status(" verify_commit: %s: source=%s target=%s", working_offset, source_rev_tmp, target_rev_tmp, level=ui.DEBUG, color='GREEN')
295 if not target_rev_tmp:
296 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.VERBOSE, color='RED')
297 raise VerificationError("Unable to find corresponding target_rev for source_rev r%s in rev_map (path_offset='%s')" % (source_rev_tmp, path_offset))
298 if target_rev_tmp not in target_revs:
299 # If found a source_rev with no equivalent target_rev in target_revs,
300 # check if the only difference in source_rev vs. source_rev-1 is the
301 # removal/addition of a trailing newline char, since this seems to get
302 # stripped-out sometimes during the replay (via "svn export"?).
303 # Strip any trailing \r\n from file-content (http://stackoverflow.com/a/1656218/346778)
304 sum1 = run_shell_command("svn cat -r %s '%s' | perl -i -p0777we's/\\r\\n\z//' | md5sum" % (source_rev_tmp, source_repos_url+working_path+"@"+str(source_rev_tmp)))
305 sum2 = run_shell_command("svn cat -r %s '%s' | perl -i -p0777we's/\\r\\n\z//' | md5sum" % (source_rev_tmp-1, source_repos_url+working_path+"@"+str(source_rev_tmp-1)))
306 if sum1 <> sum2:
307 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.VERBOSE, color='RED')
308 raise VerificationError("Found source_rev (r%s) with no corresponding target_rev: path_offset='%s'" % (source_rev_tmp, path_offset))
309 target_revs_rmndr.remove(target_rev_tmp)
310 if target_revs_rmndr:
311 rmndr_list = ", ".join(map(str, target_revs_rmndr))
312 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.VERBOSE, color='RED')
313 raise VerificationError("Found one or more *extra* target_revs: path_offset='%s', target_revs='%s'" % (path_offset, rmndr_list))
314 ui.status(" (%s/%s) Verify path: OK: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.VERBOSE)
315
316 # Ensure there are no "extra" files in the target side
317 if options.verify == 2:
318 target_paths = []
319 child_paths = run_svn(["list", "--recursive", "-r", target_rev, target_url+"@"+str(target_rev)])
320 child_paths = child_paths.strip("\n").split("\n")
321 for child_path in child_paths:
322 if not child_path:
323 continue
324 # Directories have a trailing slash in the "svn list" output
325 child_path_is_dir = True if child_path[-1] == "/" else False
326 child_path_offset = child_path.rstrip('/') if child_path_is_dir else child_path
327 if not child_path_is_dir:
328 target_paths.append(child_path_offset)
329 # Compare
330 for path_offset in target_paths:
331 if not path_offset in check_paths:
332 raise VerificationError("Path exists in target (@%s) but not source (@%s): %s" % (target_rev, source_rev, path_offset))
333 for path_offset in check_paths:
334 if not path_offset in target_paths:
335 raise VerificationError("Path exists in source (@%s) but not target (@%s): %s" % (source_rev, target_rev, path_offset))
336
337 def full_svn_revert():
338 """
339 Do an "svn revert" and proactively remove any extra files in the working copy.
340 """
341 run_svn(["revert", "--recursive", "."])
342 output = run_svn(["status"])
343 if output:
344 output_lines = output.strip("\n").split("\n")
345 for line in output_lines:
346 if line[0] == "?":
347 path = line[4:].strip(" ")
348 if os.path.isfile(path):
349 os.remove(path)
350 if os.path.isdir(path):
351 shutil.rmtree(path)
352
353 def gen_tracking_revprops(source_rev):
354 """
355 Build an array of svn2svn-specific source-tracking revprops.
356 """
357 revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
358 {'name':'svn2svn:source_url', 'value':source_url},
359 {'name':'svn2svn:source_rev', 'value':source_rev}]
360 return revprops
361
362 def sync_svn_props(source_url, source_rev, path_offset):
363 """
364 Carry-forward any unversioned properties from the source repo to the
365 target WC.
366 """
367 source_props = svnclient.get_all_props(join_path(source_url, path_offset), source_rev)
368 target_props = svnclient.get_all_props(path_offset)
369 if 'svn:mergeinfo' in source_props:
370 # Never carry-forward "svn:mergeinfo"
371 del source_props['svn:mergeinfo']
372 for prop in target_props:
373 if prop not in source_props:
374 # Remove any properties which exist in target but not source
375 run_svn(["propdel", prop, path_offset])
376 for prop in source_props:
377 if prop not in target_props or \
378 source_props[prop] != target_props[prop]:
379 # Set/update any properties which exist in source but not target or
380 # whose value differs between source vs. target.
381 run_svn(["propset", prop, source_props[prop], path_offset])
382
383 def in_svn(p, require_in_repo=False, prefix=""):
384 """
385 Check if a given file/folder is being tracked by Subversion.
386 Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
387 With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
388 Use "svn status" to check the status of the file/folder.
389 """
390 entries = svnclient.get_svn_status(p, no_recursive=True)
391 if not entries:
392 return False
393 d = entries[0]
394 if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
395 # If caller requires this path to be in the SVN repo, prevent returning True
396 # for paths that are only locally-added.
397 ret = False
398 else:
399 # Don't consider files tracked as deleted in the WC as under source-control.
400 # Consider files which are locally added/copied as under source-control.
401 ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
402 ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
403 return ret
404
405 def is_child_path(path, p_path):
406 return True if (path == p_path) or (path.startswith(p_path+"/")) else False
407
408 def join_path(base, child):
409 base.rstrip('/')
410 return base+"/"+child if child else base
411
412 def find_svn_ancestors(svn_repos_url, start_path, start_rev, stop_base_path=None, prefix=""):
413 """
414 Given an initial starting path+rev, walk the SVN history backwards to inspect the
415 ancestry of that path, optionally seeing if it traces back to stop_base_path.
416
417 Build an array of copyfrom_path and copyfrom_revision pairs for each of the "svn copy"'s.
418 If we find a copyfrom_path which stop_base_path is a substring match of (e.g. we crawled
419 back to the initial branch-copy from trunk), then return the collection of ancestor
420 paths. Otherwise, copyfrom_path has no ancestry compared to stop_base_path.
421
422 This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
423 file/folder was renamed in a branch and then that branch was merged back to trunk.
424
425 'svn_repos_url' is the full URL to the root of the SVN repository,
426 e.g. 'file:///path/to/repo'
427 'start_path' is the path in the SVN repo to the source path to start checking
428 ancestry at, e.g. '/branches/fix1/projectA/file1.txt'.
429 'start_rev' is the revision to start walking the history of start_path backwards from.
430 'stop_base_path' is the path in the SVN repo to stop tracing ancestry once we've reached,
431 i.e. the target path we're trying to trace ancestry back to, e.g. '/trunk'.
432 """
433 ui.status(prefix + ">> find_svn_ancestors: Start: (%s) start_path: %s stop_base_path: %s",
434 svn_repos_url, start_path+"@"+str(start_rev), stop_base_path, level=ui.DEBUG, color='YELLOW')
435 done = False
436 no_ancestry = False
437 cur_path = start_path
438 cur_rev = start_rev
439 first_iter_done = False
440 ancestors = []
441 while not done:
442 # Get the first "svn log" entry for cur_path (relative to @cur_rev)
443 ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url+cur_path+"@"+str(cur_rev), level=ui.DEBUG, color='YELLOW')
444 log_entry = svnclient.get_first_svn_log_entry(svn_repos_url+cur_path, 1, cur_rev)
445 if not log_entry:
446 ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
447 done = True
448 break
449 # If we found a copy-from case which matches our stop_base_path, we're done.
450 # ...but only if we've at least tried to search for the first copy-from path.
451 if stop_base_path is not None and first_iter_done and is_child_path(cur_path, stop_base_path):
452 ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(cur_path, stop_base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
453 done = True
454 break
455 first_iter_done = True
456 # Search for any actions on our target path (or parent paths).
457 changed_paths_temp = []
458 for d in log_entry['changed_paths']:
459 path = d['path']
460 if is_child_path(cur_path, path):
461 changed_paths_temp.append({'path': path, 'data': d})
462 if not changed_paths_temp:
463 # If no matches, then we've hit the end of the ancestry-chain.
464 ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
465 done = True
466 continue
467 # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
468 changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
469 # Find the action for our cur_path in this revision. Use a loop to check in reverse order,
470 # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from
471 # then we still correctly match the deepest copy-from.
472 for v in changed_paths:
473 d = v['data']
474 path = d['path']
475 # Check action-type for this file
476 action = d['action']
477 if action not in _valid_svn_actions:
478 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
479 % (log_entry['revision'], action))
480 ui.status(prefix + "> %s %s%s", action, path,
481 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
482 level=ui.DEBUG, color='YELLOW')
483 if action == 'D':
484 # If file/folder was deleted, ancestry-chain stops here
485 if stop_base_path:
486 no_ancestry = True
487 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
488 done = True
489 break
490 if action in 'RA':
491 # If file/folder was added/replaced but not a copy, ancestry-chain stops here
492 if not d['copyfrom_path']:
493 if stop_base_path:
494 no_ancestry = True
495 ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
496 "Added" if action == "A" else "Replaced",
497 level=ui.DEBUG, color='YELLOW')
498 done = True
499 break
500 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
501 # and keep checking for ancestors
502 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
503 action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
504 level=ui.DEBUG, color='YELLOW')
505 ancestors.append({'path': cur_path, 'revision': log_entry['revision'],
506 'copyfrom_path': cur_path.replace(d['path'], d['copyfrom_path']), 'copyfrom_rev': d['copyfrom_revision']})
507 cur_path = cur_path.replace(d['path'], d['copyfrom_path'])
508 cur_rev = d['copyfrom_revision']
509 # Follow the copy and keep on searching
510 break
511 if stop_base_path and no_ancestry:
512 # If we're tracing back ancestry to a specific target stop_base_path and
513 # the ancestry-chain stopped before we reached stop_base_path, then return
514 # nothing since there is no ancestry chaining back to that target.
515 ancestors = []
516 if ancestors:
517 if ui.get_level() >= ui.DEBUG:
518 max_len = 0
519 for idx in range(len(ancestors)):
520 d = ancestors[idx]
521 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
522 ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
523 for idx in range(len(ancestors)):
524 d = ancestors[idx]
525 ui.status(prefix + " [%s] %s --> %s", idx,
526 str(d['path']+"@"+str(d['revision'])).ljust(max_len),
527 str(d['copyfrom_path']+"@"+str(d['copyfrom_rev'])),
528 level=ui.DEBUG, color='YELLOW')
529 else:
530 ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
531 svn_repos_url+start_path+"@"+str(start_rev), level=ui.DEBUG, color='YELLOW')
532 return ancestors
533
534 def get_rev_map(source_rev, prefix):
535 """
536 Find the equivalent rev # in the target repo for the given rev # from the source repo.
537 """
538 ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
539 # Find the highest entry less-than-or-equal-to source_rev
540 for rev in range(int(source_rev), 0, -1):
541 in_rev_map = True if rev in rev_map else False
542 ui.status(prefix + ">> get_rev_map: rev=%s in_rev_map=%s", rev, str(in_rev_map), level=ui.DEBUG, color='BLACK_B')
543 if in_rev_map:
544 return int(rev_map[rev])
545 # Else, we fell off the bottom of the rev_map. Ruh-roh...
546 return None
547
548 def set_rev_map(source_rev, target_rev):
549 #ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
550 global rev_map
551 rev_map[int(source_rev)]=int(target_rev)
552
553 def build_rev_map(target_url, target_end_rev, source_info):
554 """
555 Check for any already-replayed history from source_url (source_info) and
556 build the mapping-table of source_rev -> target_rev.
557 """
558 global rev_map
559 rev_map = {}
560 ui.status("Rebuilding target_rev -> source_rev rev_map...", level=ui.VERBOSE)
561 proc_count = 0
562 it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
563 for log_entry in it_log_entries:
564 if log_entry['revprops']:
565 revprops = {}
566 for v in log_entry['revprops']:
567 if v['name'].startswith('svn2svn:'):
568 revprops[v['name']] = v['value']
569 if revprops and \
570 revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
571 revprops['svn2svn:source_url'] == source_info['url']:
572 source_rev = revprops['svn2svn:source_rev']
573 target_rev = log_entry['revision']
574 set_rev_map(source_rev, target_rev)
575 proc_count += 1
576 if proc_count % 500 == 0:
577 ui.status("...processed %s (%s of %s)..." % (proc_count, target_rev, target_end_rev), level=ui.VERBOSE)
578
579 def get_svn_dirlist(svn_path, rev_number = ""):
580 """
581 Get a list of all the child contents (recusive) of the given folder path.
582 """
583 args = ["list"]
584 path = svn_path
585 if rev_number:
586 args += ["-r", rev_number]
587 path += "@"+str(rev_number)
588 args += [path]
589 paths = run_svn(args, no_fail=True)
590 paths = paths.strip("\n").split("\n") if len(paths)>1 else []
591 return paths
592
593 def path_in_list(paths, path):
594 for p in paths:
595 if is_child_path(path, p):
596 return True
597 return False
598
599 def add_path(paths, path):
600 if not path_in_list(paths, path):
601 paths.append(path)
602
603 def in_ancestors(ancestors, ancestor):
604 match = True
605 for idx in range(len(ancestors)-1, 0, -1):
606 if int(ancestors[idx]['revision']) > ancestor['revision']:
607 match = is_child_path(ancestor['path'], ancestors[idx]['path'])
608 break
609 return match
610
611 def do_svn_add(source_url, path_offset, source_rev, source_ancestors, \
612 parent_copyfrom_path="", parent_copyfrom_rev="", \
613 export_paths={}, is_dir = False, skip_paths=[], prefix = ""):
614 """
615 Given the add'd source path, replay the "svn add/copy" commands to correctly
616 track renames across copy-from's.
617
618 For example, consider a sequence of events like this:
619 1. svn copy /trunk /branches/fix1
620 2. (Make some changes on /branches/fix1)
621 3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2 " Rename folder
622 4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt " Rename file inside renamed folder
623 5. svn co /trunk && svn merge /branches/fix1
624 After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
625 and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
626 to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
627 we'd lose the logical history that Proj2/file2.txt is really a descendant
628 of Proj1/file1.txt.
629
630 'path_offset' is the offset from source_base to the file to check ancestry for,
631 e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
632 'source_rev' is the revision ("svn log") that we're processing from the source repo.
633 'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
634 directory, when being called recursively by do_svn_add_dir().
635 'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
636 'is_dir' is whether path_offset is a directory (rather than a file).
637 """
638 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
639 ui.status(prefix + ">> do_svn_add: %s %s", join_path(source_base, path_offset)+"@"+str(source_rev),
640 " (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
641 level=ui.DEBUG, color='GREEN')
642 # Check if the given path has ancestors which chain back to the current source_base
643 found_ancestor = False
644 ancestors = find_svn_ancestors(source_repos_url, join_path(source_base, path_offset), source_rev, stop_base_path=source_base, prefix=prefix+" ")
645 ancestor = ancestors[len(ancestors)-1] if ancestors else None # Choose the eldest ancestor, i.e. where we reached stop_base_path=source_base
646 if ancestor and not in_ancestors(source_ancestors, ancestor):
647 ancestor = None
648 copyfrom_path = ancestor['copyfrom_path'] if ancestor else ""
649 copyfrom_rev = ancestor['copyfrom_rev'] if ancestor else ""
650 if ancestor:
651 # The copy-from path has ancestry back to source_url.
652 ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
653 level=ui.DEBUG, color='GREEN', bold=True)
654 found_ancestor = True
655 # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
656 # return None in the case where copyfrom_rev is *before* our source_start_rev.
657 tgt_rev = get_rev_map(copyfrom_rev, prefix+" ")
658 ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
659 else:
660 ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
661 found_ancestor = False
662 if found_ancestor and tgt_rev:
663 # Check if this path_offset in the target WC already has this ancestry, in which
664 # case there's no need to run the "svn copy" (again).
665 path_in_svn = in_svn(path_offset, prefix=prefix+" ")
666 log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+" ") else []
667 if (not log_entry or (log_entry['revision'] != tgt_rev)):
668 copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
669 ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
670 ui.status(prefix + " copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
671 ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
672 if path_in_svn and \
673 ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
674 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
675 # When being called recursively, if this child entry has the same ancestor as the
676 # the parent, then no need to try to run another "svn copy".
677 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
678 parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
679 pass
680 else:
681 # Copy this path from the equivalent path+rev in the target repo, to create the
682 # equivalent history.
683 if parent_copyfrom_path:
684 # If we have a parent copy-from path, we mis-match that so display a status
685 # message describing the action we're mimic'ing. If path_in_svn, then this
686 # is logically a "replace" rather than an "add".
687 ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), join_path(source_base, path_offset), ancestors[0]['copyfrom_path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
688 if path_in_svn:
689 # If local file is already under version-control, then this is a replace.
690 ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
691 run_svn(["update", path_offset])
692 run_svn(["remove", "--force", path_offset])
693 run_svn(["copy", "-r", tgt_rev, join_path(target_url, copyfrom_offset)+"@"+str(tgt_rev), path_offset])
694 if is_dir:
695 # Export the final verison of all files in this folder.
696 add_path(export_paths, path_offset)
697 else:
698 # Export the final verison of this file.
699 run_svn(["export", "--force", "-r", source_rev,
700 source_repos_url+join_path(source_base, path_offset)+"@"+str(source_rev), path_offset])
701 if options.keep_prop:
702 sync_svn_props(source_url, source_rev, path_offset)
703 else:
704 ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
705 else:
706 # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
707 # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
708 # Create (parent) directory if needed.
709 # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
710 # split-out to a shared tag?
711 p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or None
712 if p_path and not os.path.exists(p_path):
713 run_svn(["mkdir", p_path])
714 if not in_svn(path_offset, prefix=prefix+" "):
715 if is_dir:
716 # Export the final verison of all files in this folder.
717 add_path(export_paths, path_offset)
718 else:
719 # Export the final verison of this file. We *need* to do this before running
720 # the "svn add", even if we end-up re-exporting this file again via export_paths.
721 run_svn(["export", "--force", "-r", source_rev,
722 source_repos_url+join_path(source_base, path_offset)+"@"+str(source_rev), path_offset])
723 # If not already under version-control, then "svn add" this file/folder.
724 run_svn(["add", "--parents", path_offset])
725 if options.keep_prop:
726 sync_svn_props(source_url, source_rev, path_offset)
727 if is_dir:
728 # For any folders that we process, process any child contents, so that we correctly
729 # replay copies/replaces/etc.
730 do_svn_add_dir(source_url, path_offset, source_rev, source_ancestors,
731 copyfrom_path, copyfrom_rev, export_paths, skip_paths, prefix+" ")
732
733 def do_svn_add_dir(source_url, path_offset, source_rev, source_ancestors, \
734 parent_copyfrom_path, parent_copyfrom_rev, \
735 export_paths, skip_paths, prefix=""):
736 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
737 # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
738 # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
739 # associated remote repo folder. (Is this a problem?)
740 paths_local = get_svn_dirlist(path_offset)
741 paths_remote = get_svn_dirlist(join_path(source_url, path_offset), source_rev)
742 ui.status(prefix + ">> do_svn_add_dir: paths_local: %s", str(paths_local), level=ui.DEBUG, color='GREEN')
743 ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
744 # Update files/folders which exist in remote but not local
745 for path in paths_remote:
746 path_is_dir = True if path[-1] == "/" else False
747 working_path = join_path(path_offset, (path.rstrip('/') if path_is_dir else path)).lstrip('/')
748 #print "working_path:%s = path_offset:%s + path:%s" % (working_path, path_offset, path)
749 if not working_path in skip_paths:
750 do_svn_add(source_url, working_path, source_rev, source_ancestors,
751 parent_copyfrom_path, parent_copyfrom_rev,
752 export_paths, path_is_dir, skip_paths, prefix+" ")
753 # Remove files/folders which exist in local but not remote
754 for path in paths_local:
755 if not path in paths_remote:
756 path_is_dir = True if path[-1] == "/" else False
757 working_path = join_path(path_offset, (path.rstrip('/') if path_is_dir else path)).lstrip('/')
758 ui.status(" %s %s", 'D', join_path(source_base, working_path), level=ui.VERBOSE)
759 run_svn(["update", working_path])
760 run_svn(["remove", "--force", working_path])
761 # TODO: Does this handle deleted folders too? Wouldn't want to have a case
762 # where we only delete all files from folder but leave orphaned folder around.
763
764 def process_svn_log_entry(log_entry, ancestors, commit_paths, prefix = ""):
765 """
766 Process SVN changes from the given log entry. Build an array (commit_paths)
767 of the paths in the working-copy that were changed, i.e. the paths which
768 we'll pass to "svn commit".
769 """
770 export_paths = []
771 source_rev = log_entry['revision']
772 source_url = log_entry['url']
773 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
774 ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
775 for d in log_entry['changed_paths']:
776 # Get the full path for this changed_path
777 # e.g. '/branches/bug123/projectA/file1.txt'
778 path = d['path']
779 if not is_child_path(path, source_base):
780 # Ignore changed files that are not part of this subdir
781 ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
782 continue
783 if d['kind'] == "" or d['kind'] == 'none':
784 # The "kind" value was introduced in SVN 1.6, and "svn log --xml" won't return a "kind"
785 # value for commits made on a pre-1.6 repo, even if the server is now running 1.6.
786 # We need to use other methods to fetch the node-kind for these cases.
787 d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
788 assert (d['kind'] == 'file') or (d['kind'] == 'dir')
789 path_is_dir = True if d['kind'] == 'dir' else False
790 path_is_file = True if d['kind'] == 'file' else False
791 # Calculate the offset (based on source_base) for this changed_path
792 # e.g. 'projectA/file1.txt'
793 # (path = source_base + "/" + path_offset)
794 path_offset = path[len(source_base):].strip("/")
795 # Get the action for this path
796 action = d['action']
797 if action not in _valid_svn_actions:
798 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
799 % (source_rev, action))
800 ui.status(" %s %s%s", action, d['path'],
801 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
802 level=ui.VERBOSE)
803
804 # Try to be efficient and keep track of an explicit list of paths in the
805 # working copy that changed. If we commit from the root of the working copy,
806 # then SVN needs to crawl the entire working copy looking for pending changes.
807 commit_paths.append(path_offset)
808
809 # Special-handling for replace's
810 if action == 'R':
811 # If file was "replaced" (deleted then re-added, all in same revision),
812 # then we need to run the "svn rm" first, then change action='A'. This
813 # lets the normal code below handle re-"svn add"'ing the files. This
814 # should replicate the "replace".
815 if path_offset and in_svn(path_offset):
816 # Target path might not be under version-control yet, e.g. parent "add"
817 # was a copy-from a branch which had no ancestry back to trunk, and each
818 # child folder under that parent folder is a "replace" action on the final
819 # merge to trunk. Since the child folders will be in skip_paths, do_svn_add
820 # wouldn't have created them while processing the parent "add" path.
821 if path_is_dir:
822 # Need to "svn update" before "svn remove" in case child contents are at
823 # a higher rev than the (parent) path_offset.
824 run_svn(["update", path_offset])
825 run_svn(["remove", "--force", path_offset])
826 action = 'A'
827
828 # Handle all the various action-types
829 # (Handle "add" first, for "svn copy/move" support)
830 if action == 'A':
831 # Determine where to export from.
832 svn_copy = False
833 # Handle cases where this "add" was a copy from another URL in the source repo
834 if d['copyfrom_revision']:
835 copyfrom_path = d['copyfrom_path']
836 copyfrom_rev = d['copyfrom_revision']
837 skip_paths = []
838 for tmp_d in log_entry['changed_paths']:
839 tmp_path = tmp_d['path']
840 if is_child_path(tmp_path, path) and tmp_d['action'] in 'ARD':
841 # Build list of child entries which are also in the changed_paths list,
842 # so that do_svn_add() can skip processing these entries when recursing
843 # since we'll end-up processing them later. Don't include action="M" paths
844 # in this list because it's non-conclusive: it could just mean that the
845 # file was modified *after* the copy-from, so we still want do_svn_add()
846 # to re-create the correct ancestry.
847 tmp_path_offset = tmp_path[len(source_base):].strip("/")
848 skip_paths.append(tmp_path_offset)
849 do_svn_add(source_url, path_offset, source_rev, ancestors, "", "", export_paths, path_is_dir, skip_paths, prefix+" ")
850 # Else just "svn export" the files from the source repo and "svn add" them.
851 else:
852 # Create (parent) directory if needed
853 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or None
854 if p_path and not os.path.exists(p_path):
855 run_svn(["mkdir", p_path])
856 # Export the entire added tree.
857 if path_is_dir:
858 # For directories, defer the (recurisve) "svn export". Might have a
859 # situation in a branch merge where the entry in the svn-log is a
860 # non-copy-from'd "add" but there are child contents (that we haven't
861 # gotten to yet in log_entry) that are copy-from's. When we try do
862 # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
863 # having pre-existing (svn-add'd) contents creates some trouble.
864 # Instead, just create the stub folders ("svn mkdir" above) and defer
865 # exporting the final file-state until the end.
866 add_path(export_paths, path_offset)
867 else:
868 # Export the final verison of this file. We *need* to do this before running
869 # the "svn add", even if we end-up re-exporting this file again via export_paths.
870 run_svn(["export", "--force", "-r", source_rev,
871 join_path(source_url, path_offset)+"@"+str(source_rev), path_offset])
872 if not in_svn(path_offset, prefix=prefix+" "):
873 # Need to use in_svn here to handle cases where client committed the parent
874 # folder and each indiv sub-folder.
875 run_svn(["add", "--parents", path_offset])
876 if options.keep_prop:
877 sync_svn_props(source_url, source_rev, path_offset)
878
879 elif action == 'D':
880 if path_is_dir:
881 # For dirs, need to "svn update" before "svn remove" because the final
882 # "svn commit" will fail if the parent (path_offset) is at a lower rev
883 # than any of the child contents. This needs to be a recursive update.
884 run_svn(["update", path_offset])
885 run_svn(["remove", "--force", path_offset])
886
887 elif action == 'M':
888 if path_is_file:
889 run_svn(["export", "--force", "-N" , "-r", source_rev,
890 join_path(source_url, path_offset)+"@"+str(source_rev), path_offset])
891 if path_is_dir:
892 # For dirs, need to "svn update" before export/prop-sync because the
893 # final "svn commit" will fail if the parent is at a lower rev than
894 # child contents. Just need to update the rev-state of the dir (d['path']),
895 # don't need to recursively update all child contents.
896 # (??? is this the right reason?)
897 run_svn(["update", "-N", path_offset])
898 if options.keep_prop:
899 sync_svn_props(source_url, source_rev, path_offset)
900
901 else:
902 raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
903 % action)
904
905 # Export the final version of all add'd paths from source_url
906 if export_paths:
907 for path_offset in export_paths:
908 run_svn(["export", "--force", "-r", source_rev,
909 join_path(source_url, path_offset)+"@"+str(source_rev), path_offset])
910
911 def keep_revnum(source_rev, target_rev_last, wc_target_tmp):
912 """
913 Add "padding" target revisions as needed to keep source and target
914 revision #'s identical.
915 """
916 if int(source_rev) <= int(target_rev_last):
917 raise InternalError("keep-revnum mode is enabled, "
918 "but source revision (r%s) is less-than-or-equal last target revision (r%s)" % \
919 (source_rev, target_rev_last))
920 if int(target_rev_last) < int(source_rev)-1:
921 # Add "padding" target revisions to keep source and target rev #'s identical
922 if os.path.exists(wc_target_tmp):
923 shutil.rmtree(wc_target_tmp)
924 run_svn(["checkout", "-r", "HEAD", "--depth=empty", target_repos_url, wc_target_tmp])
925 for rev_num in range(int(target_rev_last)+1, int(source_rev)):
926 run_svn(["propset", "svn2svn:keep-revnum", rev_num, wc_target_tmp])
927 output = run_svn(["commit", "-m", "", wc_target_tmp])
928 rev_num_tmp = parse_svn_commit_rev(output) if output else None
929 assert rev_num == rev_num_tmp
930 ui.status("Committed revision %s (keep-revnum).", rev_num)
931 target_rev_last = rev_num
932 shutil.rmtree(wc_target_tmp)
933 return target_rev_last
934
935 def disp_svn_log_summary(log_entry):
936 ui.status("------------------------------------------------------------------------", level=ui.VERBOSE)
937 ui.status("r%s | %s | %s",
938 log_entry['revision'],
939 log_entry['author'],
940 str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')), level=ui.VERBOSE)
941 ui.status(log_entry['message'], level=ui.VERBOSE)
942
943 def real_main(args):
944 global source_url, target_url, rev_map
945 source_url = urllib.quote(args.pop(0).rstrip("/"),"/:") # e.g. 'http://server/svn/source/trunk'
946 target_url = urllib.quote(args.pop(0).rstrip("/"),"/:") # e.g. 'file:///svn/target/trunk'
947 ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
948
949 # Make sure that both the source and target URL's are valid
950 source_info = svnclient.get_svn_info(source_url)
951 assert is_child_path(source_url, source_info['repos_url'])
952 target_info = svnclient.get_svn_info(target_url)
953 assert is_child_path(target_url, target_info['repos_url'])
954
955 # Init global vars
956 global source_repos_url,source_base,source_repos_uuid
957 source_repos_url = source_info['repos_url'] # e.g. 'http://server/svn/source'
958 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
959 source_repos_uuid = source_info['repos_uuid']
960 global target_repos_url,target_base
961 target_repos_url = target_info['repos_url'] # e.g. 'http://server/svn/target'
962 target_base = target_url[len(target_repos_url):] # e.g. '/trunk'
963
964 # Init start and end revision
965 try:
966 source_start_rev = svnclient.get_svn_rev(source_repos_url, options.rev_start if options.rev_start else 1)
967 except ExternalCommandFailed:
968 print "Error: Invalid start source revision value: %s" % (options.rev_start)
969 sys.exit(1)
970 try:
971 source_end_rev = svnclient.get_svn_rev(source_repos_url, options.rev_end if options.rev_end else "HEAD")
972 except ExternalCommandFailed:
973 print "Error: Invalid end source revision value: %s" % (options.rev_end)
974 sys.exit(1)
975 ui.status("Using source revision range %s:%s", source_start_rev, source_end_rev, level=ui.VERBOSE)
976
977 # TODO: If options.keep_date, should we try doing a "svn propset" on an *existing* revision
978 # as a sanity check, so we check if the pre-revprop-change hook script is correctly setup
979 # before doing first replay-commit?
980
981 target_rev_last = target_info['revision'] # Last revision # in the target repo
982 wc_target = os.path.abspath('_wc_target')
983 wc_target_tmp = os.path.abspath('_wc_target_tmp')
984 num_entries_proc = 0
985 commit_count = 0
986 source_rev = None
987 target_rev = None
988
989 # Check out a working copy of target_url if needed
990 wc_exists = os.path.exists(wc_target)
991 if wc_exists and not options.cont_from_break:
992 shutil.rmtree(wc_target)
993 wc_exists = False
994 if not wc_exists:
995 ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
996 svnclient.svn_checkout(target_url, wc_target)
997 os.chdir(wc_target)
998 if wc_exists:
999 # If using an existing WC, make sure it's clean ("svn revert")
1000 ui.status("Cleaning-up _wc_target...", level=ui.VERBOSE)
1001 run_svn(["cleanup"])
1002 full_svn_revert()
1003
1004 if not options.cont_from_break:
1005 # Warn user if trying to start (non-continue) into a non-empty target path
1006 if not options.force_nocont:
1007 top_paths = run_svn(["list", "-r", "HEAD", target_url])
1008 if len(top_paths)>0:
1009 print "Error: Trying to replay (non-continue-mode) into a non-empty target_url location. " \
1010 "Use --force if you're sure this is what you want."
1011 sys.exit(1)
1012 # Get the first log entry at/after source_start_rev, which is where
1013 # we'll do the initial import from.
1014 source_ancestors = find_svn_ancestors(source_repos_url, source_base, source_end_rev, prefix=" ")
1015 it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False, ancestors=source_ancestors)
1016 source_start_log = None
1017 for log_entry in it_log_start:
1018 # Pick the first entry. Need to use a "for ..." loop since we're using an iterator.
1019 source_start_log = log_entry
1020 break
1021 if not source_start_log:
1022 raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
1023 (source_start_rev, source_end_rev, source_url))
1024
1025 # This is the revision we will start from for source_url
1026 source_start_rev = int(source_start_log['revision'])
1027 ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
1028 ui.status("", level=ui.VERBOSE)
1029 if options.keep_revnum and source_rev > target_rev_last:
1030 target_rev_last = keep_revnum(source_rev, target_rev_last, wc_target_tmp)
1031
1032 # For the initial commit to the target URL, export all the contents from
1033 # the source URL at the start-revision.
1034 disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_repos_url, source_start_rev, source_start_rev))
1035 # Export and add file-contents from source_url@source_start_rev
1036 source_start_url = source_url if not source_ancestors else source_repos_url+source_ancestors[len(source_ancestors)-1]['copyfrom_path']
1037 top_paths = run_svn(["list", "-r", source_start_rev, source_start_url+"@"+str(source_start_rev)])
1038 top_paths = top_paths.strip("\n").split("\n")
1039 for path in top_paths:
1040 # For each top-level file/folder...
1041 if not path:
1042 continue
1043 # Directories have a trailing slash in the "svn list" output
1044 path_is_dir = True if path[-1] == "/" else False
1045 path_offset = path.rstrip('/') if path_is_dir else path
1046 if in_svn(path_offset, prefix=" "):
1047 raise InternalError("Cannot replay history on top of pre-existing structure: %s" % join_path(source_start_url, path_offset))
1048 if path_is_dir and not os.path.exists(path_offset):
1049 os.makedirs(path_offset)
1050 run_svn(["export", "--force", "-r" , source_start_rev, join_path(source_start_url, path_offset)+"@"+str(source_start_rev), path_offset])
1051 run_svn(["add", path_offset])
1052 # Update any properties on the newly added content
1053 paths = run_svn(["list", "--recursive", "-r", source_start_rev, source_start_url+"@"+str(source_start_rev)])
1054 paths = paths.strip("\n").split("\n")
1055 if options.keep_prop:
1056 sync_svn_props(source_start_url, source_start_rev, "")
1057 for path in paths:
1058 if not path:
1059 continue
1060 # Directories have a trailing slash in the "svn list" output
1061 path_is_dir = True if path[-1] == "/" else False
1062 path_offset = path.rstrip('/') if path_is_dir else path
1063 ui.status(" A %s", join_path(source_base, path_offset), level=ui.VERBOSE)
1064 if options.keep_prop:
1065 sync_svn_props(source_start_url, source_start_rev, path_offset)
1066 # Commit the initial import
1067 num_entries_proc += 1
1068 target_revprops = gen_tracking_revprops(source_start_rev) # Build source-tracking revprop's
1069 target_rev = commit_from_svn_log_entry(source_start_log, target_revprops=target_revprops)
1070 if target_rev:
1071 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
1072 set_rev_map(source_start_rev, target_rev)
1073 commit_count += 1
1074 target_rev_last = target_rev
1075 if options.verify:
1076 verify_commit(source_rev, target_rev_last)
1077 else:
1078 # Re-build the rev_map based on any already-replayed history in target_url
1079 build_rev_map(target_url, target_rev_last, source_info)
1080 if not rev_map:
1081 print "Error: Called with continue-mode, but no already-replayed source history found in target_url."
1082 sys.exit(1)
1083 source_start_rev = int(max(rev_map, key=rev_map.get))
1084 assert source_start_rev
1085 ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
1086 ui.status("", level=ui.VERBOSE)
1087
1088 if options.keep_revnum and source_start_rev < target_rev_last:
1089 print "Error: Last target revision (r%s) is equal-or-higher than starting source revision (r%s). " \
1090 "Cannot use --keep-revnum mode." % (target_rev_last, source_start_rev)
1091 sys.exit(1)
1092
1093 svn_vers_t = svnclient.get_svn_client_version()
1094 svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
1095
1096 # Load SVN log starting from source_start_rev + 1
1097 source_ancestors = find_svn_ancestors(source_repos_url, source_base, source_end_rev, prefix=" ")
1098 it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True, ancestors=source_ancestors) if source_start_rev < source_end_rev else []
1099 source_rev = None
1100
1101 try:
1102 for log_entry in it_log_entries:
1103 if options.entries_proc_limit:
1104 if num_entries_proc >= options.entries_proc_limit:
1105 break
1106 # Replay this revision from source_url into target_url
1107 source_rev = log_entry['revision']
1108 log_url = log_entry['url']
1109 #print "source_url:%s log_url:%s" % (source_url, log_url)
1110 if options.keep_revnum:
1111 target_rev_last = keep_revnum(source_rev, target_rev_last, wc_target_tmp)
1112 disp_svn_log_summary(log_entry)
1113 # Process all the changed-paths in this log entry
1114 commit_paths = []
1115 process_svn_log_entry(log_entry, source_ancestors, commit_paths)
1116 num_entries_proc += 1
1117 # Commit any changes made to _wc_target
1118 target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's
1119 target_rev = commit_from_svn_log_entry(log_entry, commit_paths, target_revprops=target_revprops)
1120 if target_rev:
1121 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
1122 source_rev = log_entry['revision']
1123 set_rev_map(source_rev, target_rev)
1124 target_rev_last = target_rev
1125 commit_count += 1
1126 if options.verify:
1127 verify_commit(source_rev, target_rev_last, log_entry)
1128 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
1129 if svn_vers >= 1.7 and (commit_count % 100 == 0):
1130 run_svn(["cleanup"])
1131 if not source_rev:
1132 # If there were no new source_url revisions to process, init source_rev
1133 # for the "finally" message below to be the last source revision replayed.
1134 source_rev = source_start_rev
1135 if options.verify:
1136 verify_commit(source_start_rev, target_rev_last)
1137
1138 except KeyboardInterrupt:
1139 print "\nStopped by user."
1140 print "\nCleaning-up..."
1141 run_svn(["cleanup"])
1142 full_svn_revert()
1143 except:
1144 print "\nCommand failed with following error:\n"
1145 traceback.print_exc()
1146 print "\nCleaning-up..."
1147 run_svn(["cleanup"])
1148 print run_svn(["status"])
1149 full_svn_revert()
1150 finally:
1151 print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
1152
1153 def main():
1154 # Defined as entry point. Must be callable without arguments.
1155 usage = "svn2svn, version %s\n" % str(full_version) + \
1156 "<http://nynim.org/projects/svn2svn> <https://github.com/tonyduckles/svn2svn>\n\n" + \
1157 "Usage: %prog [OPTIONS] source_url target_url\n"
1158 description = """\
1159 Replicate (replay) history from one SVN repository to another. Maintain
1160 logical ancestry wherever possible, so that 'svn log' on the replayed repo
1161 will correctly follow file/folder renames.
1162
1163 Examples:
1164 Create a copy of only /trunk from source repo, starting at r5000
1165 $ svnadmin create /svn/target
1166 $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
1167 $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
1168 1. The target_url will be checked-out to ./_wc_target
1169 2. The first commit to http://server/source/trunk at/after r5000 will be
1170 exported & added into _wc_target
1171 3. All revisions affecting http://server/source/trunk (starting at r5000)
1172 will be replayed to _wc_target. Any add/copy/move/replaces that are
1173 copy-from'd some path outside of /trunk (e.g. files renamed on a
1174 /branch and branch was merged into /trunk) will correctly maintain
1175 logical ancestry where possible.
1176
1177 Use continue-mode (-c) to pick-up where the last run left-off
1178 $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
1179 1. The target_url will be checked-out to ./_wc_target, if not already
1180 checked-out
1181 2. All new revisions affecting http://server/source/trunk starting from
1182 the last replayed revision to file:///svn/target/trunk (based on the
1183 svn2svn:* revprops) will be replayed to _wc_target, maintaining all
1184 logical ancestry where possible."""
1185 parser = optparse.OptionParser(usage, description=description,
1186 formatter=HelpFormatter(), version="%prog "+str(full_version))
1187 parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
1188 help="enable additional output (use -vv or -vvv for more)")
1189 parser.add_option("-a", "--archive", action="store_true", dest="archive", default=False,
1190 help="archive/mirror mode; same as -UDP (see REQUIRE's below)\n"
1191 "maintain same commit author, same commit time, and file/dir properties")
1192 parser.add_option("-U", "--keep-author", action="store_true", dest="keep_author", default=False,
1193 help="maintain same commit authors (svn:author) as source\n"
1194 "(REQUIRES 'pre-revprop-change' hook script to allow 'svn:author' changes)")
1195 parser.add_option("-D", "--keep-date", action="store_true", dest="keep_date", default=False,
1196 help="maintain same commit time (svn:date) as source\n"
1197 "(REQUIRES 'pre-revprop-change' hook script to allow 'svn:date' changes)")
1198 parser.add_option("-P", "--keep-prop", action="store_true", dest="keep_prop", default=False,
1199 help="maintain same file/dir SVN properties as source")
1200 parser.add_option("-R", "--keep-revnum", action="store_true", dest="keep_revnum", default=False,
1201 help="maintain same rev #'s as source. creates placeholder target "
1202 "revisions (by modifying a 'svn2svn:keep-revnum' property at the root of the target repo)")
1203 parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
1204 help="continue from last source commit to target (based on svn2svn:* revprops)")
1205 parser.add_option("-f", "--force", action="store_true", dest="force_nocont",
1206 help="allow replaying into a non-empty target folder")
1207 parser.add_option("-r", "--revision", type="string", dest="revision", metavar="ARG",
1208 help="revision range to replay from source_url\n"
1209 "A revision argument can be one of:\n"
1210 " START start rev # (end will be 'HEAD')\n"
1211 " START:END start and ending rev #'s\n"
1212 "Any revision # formats which SVN understands are "
1213 "supported, e.g. 'HEAD', '{2010-01-31}', etc.")
1214 parser.add_option("-u", "--log-author", action="store_true", dest="log_author", default=False,
1215 help="append source commit author to replayed commit mesages")
1216 parser.add_option("-d", "--log-date", action="store_true", dest="log_date", default=False,
1217 help="append source commit time to replayed commit messages")
1218 parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
1219 help="maximum number of source revisions to process")
1220 parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
1221 help="process next source revision but don't commit changes to "
1222 "target working-copy (forces --limit=1)")
1223 parser.add_option("-x", "--verify", action="store_const", const=1, dest="verify",
1224 help="verify ancestry and content for changed paths in commit after every target commit or last target commit")
1225 parser.add_option("-X", "--verify-all", action="store_const", const=2, dest="verify",
1226 help="verify ancestry and content for entire target_url tree after every target commit or last target commit")
1227 parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
1228 help="enable debugging output (same as -vvv)")
1229 global options
1230 options, args = parser.parse_args()
1231 if len(args) != 2:
1232 parser.error("incorrect number of arguments")
1233 if options.verbosity < 10:
1234 # Expand multiple "-v" arguments to a real ui._level value
1235 options.verbosity *= 10
1236 if options.dry_run:
1237 # When in dry-run mode, only try to process the next log_entry
1238 options.entries_proc_limit = 1
1239 options.rev_start = None
1240 options.rev_end = None
1241 if options.revision:
1242 # Reg-ex for matching a revision arg (http://svnbook.red-bean.com/en/1.5/svn.tour.revs.specifiers.html#svn.tour.revs.dates)
1243 rev_patt = '[0-9A-Z]+|\{[0-9A-Za-z/\\ :-]+\}'
1244 rev = None
1245 match = re.match('^('+rev_patt+'):('+rev_patt+')$', options.revision) # First try start:end match
1246 if match is None: match = re.match('^('+rev_patt+')$', options.revision) # Next, try start match
1247 if match is None:
1248 parser.error("unexpected --revision argument format; see 'svn help log' for valid revision formats")
1249 rev = match.groups()
1250 options.rev_start = rev[0] if len(rev)>0 else None
1251 options.rev_end = rev[1] if len(rev)>1 else None
1252 if options.archive:
1253 options.keep_author = True
1254 options.keep_date = True
1255 options.keep_prop = True
1256 ui.update_config(options)
1257 return real_main(args)
1258
1259
1260 if __name__ == "__main__":
1261 sys.exit(main() or 0)