]> Tony Duckles's Git Repositories (git.nynim.org) - svn2svn.git/blob - svn2svn/run/svn2svn.py
svn2svn v1.6.0
[svn2svn.git] / svn2svn / run / svn2svn.py
1 """
2 Replicate (replay) changesets from one SVN repository to another.
3 """
4
5 from .. import base_version, full_version
6 from .. import ui
7 from .. import shell
8 from .. import svnclient
9 from ..shell import run_svn,run_shell_command
10 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
11 from parse import HelpFormatter
12 from breakhandler import BreakHandler
13
14 import sys
15 import os
16 import traceback
17 import operator
18 import optparse
19 import re
20 import urllib
21 from datetime import datetime
22
23 _valid_svn_actions = "MARD" # The list of known SVN action abbr's, from "svn log"
24
25 # Module-level variables/parameters
26 source_url = "" # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
27 source_repos_url = "" # URL to root of source SVN repo, e.g. 'http://server/svn/source'
28 source_base = "" # Relative path of source_url in source SVN repo, e.g. '/trunk'
29 source_repos_uuid = "" # UUID of source SVN repo
30 target_url ="" # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
31 target_repos_url = "" # URL to root of target SVN repo, e.g. 'http://server/svn/target'
32 target_base = "" # Relative path of target_url in target SVN repo, e.g. '/trunk'
33 rev_map = {} # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
34 options = None # optparser options
35
36 def parse_svn_commit_rev(output):
37 """
38 Parse the revision number from the output of "svn commit".
39 """
40 output_lines = output.strip("\n").split("\n")
41 rev_num = None
42 for line in output_lines:
43 if line[0:19] == 'Committed revision ':
44 rev_num = line[19:].rstrip('.')
45 break
46 assert rev_num is not None
47 return int(rev_num)
48
49 def commit_from_svn_log_entry(log_entry, commit_paths=None, target_revprops=None):
50 """
51 Given an SVN log entry and an optional list of changed paths, do an svn commit.
52 """
53 # TODO: Run optional external shell hook here, for doing pre-commit filtering
54 # Display the _wc_target "svn status" info if running in -vv (or higher) mode
55 if ui.get_level() >= ui.EXTRA:
56 ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
57 ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
58 # This will use the local timezone for displaying commit times
59 timestamp = int(log_entry['date'])
60 svn_date = str(datetime.fromtimestamp(timestamp))
61 # Uncomment this one one if you prefer UTC commit times
62 #svn_date = "%d 0" % timestamp
63 args = ["commit", "--force-log"]
64 message = log_entry['message']
65 if options.log_date:
66 message += "\nDate: " + svn_date
67 if options.log_author:
68 message += "\nAuthor: " + log_entry['author']
69 args += ["-m", message]
70 revprops = {}
71 if log_entry['revprops']:
72 # Carry forward any revprop's from the source revision
73 for v in log_entry['revprops']:
74 revprops[v['name']] = v['value']
75 if target_revprops:
76 # Add any extra revprop's we want to set for the target repo commits
77 for v in target_revprops:
78 revprops[v['name']] = v['value']
79 if revprops:
80 for key in revprops:
81 args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
82 if commit_paths:
83 if len(commit_paths)<100:
84 # If we don't have an excessive amount of individual changed paths, pass
85 # those to the "svn commit" command. Else, pass nothing so we commit at
86 # the root of the working-copy.
87 for c_path in commit_paths:
88 args += [svnclient.safe_path(c_path)]
89 rev_num = None
90 if not options.dry_run:
91 # Use BreakHandler class to temporarily redirect SIGINT handler, so that
92 # "svn commit" + post-commit rev-prop updating is a quasi-atomic unit.
93 # If user presses Ctrl-C during this, wait until after this full action
94 # has finished raising the KeyboardInterrupt exception.
95 bh = BreakHandler()
96 bh.enable()
97 # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
98 output = run_svn(args)
99 rev_num = parse_svn_commit_rev(output) if output else None
100 if rev_num is not None:
101 if options.keep_date:
102 run_svn(["propset", "--revprop", "-r", rev_num, "svn:date", log_entry['date_raw']])
103 if options.keep_author:
104 run_svn(["propset", "--revprop", "-r", rev_num, "svn:author", log_entry['author']])
105 ui.status("Committed revision %s (source r%s).", rev_num, log_entry['revision'])
106 bh.disable()
107 # Check if the user tried to press Ctrl-C
108 if bh.trapped:
109 raise KeyboardInterrupt
110 return rev_num
111
112 def verify_commit(source_rev, target_rev, log_entry=None):
113 """
114 Compare the ancestry/content/properties between source_url vs target_url
115 for a given revision.
116 """
117 error_cnt = 0
118 # Gather the offsets in the source repo to check
119 check_paths = []
120 remove_paths = []
121 # TODO: Need to make this ancestry aware
122 if options.verify == 1 and log_entry is not None: # Changed only
123 ui.status("Verifying source revision %s (only-changed)...", source_rev, level=ui.VERBOSE)
124 for d in log_entry['changed_paths']:
125 path = d['path']
126 if not is_child_path(path, source_base):
127 continue
128 if d['kind'] == "":
129 d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
130 assert (d['kind'] == 'file') or (d['kind'] == 'dir')
131 path_is_dir = True if d['kind'] == 'dir' else False
132 path_is_file = True if d['kind'] == 'file' else False
133 path_offset = path[len(source_base):].strip("/")
134 if d['action'] == 'D':
135 remove_paths.append(path_offset)
136 elif not path_offset in check_paths:
137 ui.status("verify_commit: path [mode=changed]: kind=%s: %s", d['kind'], path, level=ui.DEBUG, color='YELLOW')
138 if path_is_file:
139 ui.status(" "+"verify_commit [mode=changed]: check_paths.append('%s')", path_offset, level=ui.DEBUG, color='GREEN')
140 check_paths.append(path_offset)
141 if path_is_dir:
142 if not d['action'] in 'AR':
143 continue
144 child_paths = svnclient.list(source_url.rstrip("/")+"/"+path_offset, source_rev, recursive=True)
145 for p in child_paths:
146 child_path_is_dir = True if p['kind'] == 'dir' else False
147 child_path_offset = p['path']
148 if not child_path_is_dir:
149 # Only check files
150 working_path = (path_offset+"/" if path_offset else "") + child_path_offset
151 if not working_path in check_paths:
152 ui.status(" "+"verify_commit [mode=changed]: check_paths.append('%s'+'/'+'%s')", path_offset, child_path_offset, level=ui.DEBUG, color='GREEN')
153 check_paths.append(working_path)
154 if options.verify == 2: # All paths
155 ui.status("Verifying source revision %s (all)...", source_rev, level=ui.VERBOSE)
156 child_paths = svnclient.list(source_url, source_rev, recursive=True)
157 for p in child_paths:
158 child_path_is_dir = True if p['kind'] == 'dir' else False
159 child_path_offset = p['path']
160 if not child_path_is_dir:
161 # Only check files
162 ui.status("verify_commit [mode=all]: check_paths.append('%s')", child_path_offset, level=ui.DEBUG, color='GREEN')
163 check_paths.append(child_path_offset)
164
165 # If there were any paths deleted in the last revision (options.verify=1 mode),
166 # check that they were correctly deleted.
167 if remove_paths:
168 count_total = len(remove_paths)
169 count = 0
170 for path_offset in remove_paths:
171 count += 1
172 if in_svn(path_offset):
173 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA, color='RED')
174 ui.status("VerificationError: Path removed in source rev r%s, but still exists in target WC: %s", source_rev, path_offset, color='RED')
175 error_cnt +=1
176 else:
177 ui.status(" (%s/%s) Verify remove: OK: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA)
178
179 # Compare each of the check_path entries between source vs. target
180 if check_paths:
181 source_rev_first = int(min(rev_map, key=rev_map.get)) or 1 # The first source_rev we replayed into target
182 ui.status("verify_commit: source_rev_first:%s", source_rev_first, level=ui.DEBUG, color='YELLOW')
183 count_total = len(check_paths)
184 count = 0
185 for path_offset in check_paths:
186 count += 1
187 if count % 500 == 0:
188 ui.status("...processed %s (%s of %s)..." % (count, count, count_total), level=ui.VERBOSE)
189 ui.status("verify_commit: path_offset:%s", path_offset, level=ui.DEBUG, color='YELLOW')
190 source_log_entries = svnclient.run_svn_log(source_url.rstrip("/")+"/"+path_offset, source_rev, 1, source_rev-source_rev_first+1)
191 target_log_entries = svnclient.run_svn_log(target_url.rstrip("/")+"/"+path_offset, target_rev, 1, target_rev)
192 # Build a list of commits in source_log_entries which matches our
193 # target path_offset.
194 working_path = source_base+"/"+path_offset
195 source_revs = []
196 for log_entry in source_log_entries:
197 source_rev_tmp = log_entry['revision']
198 if source_rev_tmp < source_rev_first:
199 # Only process source revisions which have been replayed into target
200 break
201 #ui.status(" [verify_commit] source_rev_tmp:%s, working_path:%s\n%s", source_rev_tmp, working_path, pp.pformat(log_entry), level=ui.DEBUG, color='MAGENTA')
202 changed_paths_temp = []
203 for d in log_entry['changed_paths']:
204 path = d['path']
205 # Match working_path or any parents
206 if is_child_path(working_path, path):
207 ui.status(" verify_commit: changed_path: %s %s@%s (parent:%s)", d['action'], path, source_rev_tmp, working_path, level=ui.DEBUG, color='YELLOW')
208 changed_paths_temp.append({'path': path, 'data': d})
209 assert changed_paths_temp
210 # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
211 changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
212 # Find the action for our working_path in this revision. Use a loop to check in reverse order,
213 # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
214 working_path_next = working_path
215 match_d = {}
216 for v in changed_paths:
217 d = v['data']
218 if not match_d:
219 match_d = d
220 path = d['path']
221 if d['action'] not in _valid_svn_actions:
222 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
223 % (log_entry['revision'], d['action']))
224 if d['action'] in 'AR' and d['copyfrom_revision']:
225 # If we found a copy-from action for a parent path, adjust our
226 # working_path to follow the rename/copy-from, just like find_svn_ancestors().
227 working_path_next = working_path.replace(d['path'], d['copyfrom_path'])
228 match_d = d
229 break
230 if is_child_path(working_path, source_base):
231 # Only add source_rev's where the path changed in this revision was a child
232 # of source_base, so that we silently ignore any history that happened on
233 # non-source_base paths (e.g. ignore branch history if we're only replaying trunk).
234 is_diff = False
235 d = match_d
236 if d['action'] == 'M':
237 # For action="M", we need to throw out cases where the only change was to
238 # a property which we ignore, e.g. "svn:mergeinfo".
239 if d['kind'] == "":
240 d['kind'] = svnclient.get_kind(source_repos_url, working_path, log_entry['revision'], d['action'], log_entry['changed_paths'])
241 assert (d['kind'] == 'file') or (d['kind'] == 'dir')
242 if d['kind'] == 'file':
243 # Check for file-content changes
244 # TODO: This should be made ancestor-aware, since the file won't always be at the same path in rev-1
245 sum1 = run_shell_command("svn cat -r %s '%s' | md5sum" % (source_rev_tmp, source_repos_url+working_path+"@"+str(source_rev_tmp)))
246 sum2 = run_shell_command("svn cat -r %s '%s' | md5sum" % (source_rev_tmp-1, source_repos_url+working_path_next+"@"+str(source_rev_tmp-1)))
247 is_diff = True if sum1 <> sum2 else False
248 if not is_diff:
249 # Check for property changes
250 props1 = svnclient.propget_all(source_repos_url+working_path, source_rev_tmp)
251 props2 = svnclient.propget_all(source_repos_url+working_path_next, source_rev_tmp-1)
252 # Ignore changes to "svn:mergeinfo", since we don't copy that
253 if 'svn:mergeinfo' in props1: del props1['svn:mergeinfo']
254 if 'svn:mergeinfo' in props2: del props2['svn:mergeinfo']
255 for prop in props1:
256 if prop not in props2 or \
257 props1[prop] != props2[prop]:
258 is_diff = True
259 break
260 for prop in props2:
261 if prop not in props1 or \
262 props1[prop] != props2[prop]:
263 is_diff = True
264 break
265 if not is_diff:
266 ui.status(" verify_commit: skip %s@%s", working_path, source_rev_tmp, level=ui.DEBUG, color='GREEN_B', bold=True)
267 else:
268 is_diff = True
269 if is_diff:
270 ui.status(" verify_commit: source_revs.append(%s), working_path:%s", source_rev_tmp, working_path, level=ui.DEBUG, color='GREEN_B')
271 source_revs.append({'path': working_path, 'revision': source_rev_tmp})
272 working_path = working_path_next
273 # Build a list of all the target commits "svn log" returned
274 target_revs = []
275 target_revs_rmndr = []
276 for log_entry in target_log_entries:
277 target_rev_tmp = log_entry['revision']
278 ui.status(" verify_commit: target_revs.append(%s)", target_rev_tmp, level=ui.DEBUG, color='GREEN_B')
279 target_revs.append(target_rev_tmp)
280 target_revs_rmndr.append(target_rev_tmp)
281 # Compare the two lists
282 for d in source_revs:
283 working_path = d['path']
284 source_rev_tmp = d['revision']
285 target_rev_tmp = get_rev_map(source_rev_tmp, " ")
286 working_offset = working_path[len(source_base):].strip("/")
287 sum1 = run_shell_command("svn cat -r %s '%s' | md5sum" % (source_rev_tmp, source_repos_url+working_path+"@"+str(source_rev_tmp)))
288 sum2 = run_shell_command("svn cat -r %s '%s' | md5sum" % (target_rev_tmp, target_url+"/"+working_offset+"@"+str(target_rev_tmp))) if target_rev_tmp is not None else ""
289 #print "source@%s: %s" % (str(source_rev_tmp).ljust(6), sum1)
290 #print "target@%s: %s" % (str(target_rev_tmp).ljust(6), sum2)
291 ui.status(" verify_commit: %s: source=%s target=%s", working_offset, source_rev_tmp, target_rev_tmp, level=ui.DEBUG, color='GREEN')
292 if not target_rev_tmp:
293 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA, color='RED')
294 ui.status("VerificationError: Unable to find corresponding target_rev for source_rev r%s in rev_map (path_offset='%s')", source_rev_tmp, path_offset, color='RED')
295 error_cnt +=1
296 continue
297 if target_rev_tmp not in target_revs:
298 # If found a source_rev with no equivalent target_rev in target_revs,
299 # check if the only difference in source_rev vs. source_rev-1 is the
300 # removal/addition of a trailing newline char, since this seems to get
301 # stripped-out sometimes during the replay (via "svn export"?).
302 # Strip any trailing \r\n from file-content (http://stackoverflow.com/a/1656218/346778)
303 sum1 = run_shell_command("svn cat -r %s '%s' | perl -i -p0777we's/\\r\\n\z//' | md5sum" % (source_rev_tmp, source_repos_url+working_path+"@"+str(source_rev_tmp)))
304 sum2 = run_shell_command("svn cat -r %s '%s' | perl -i -p0777we's/\\r\\n\z//' | md5sum" % (source_rev_tmp-1, source_repos_url+working_path+"@"+str(source_rev_tmp-1)))
305 if sum1 <> sum2:
306 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA, color='RED')
307 ui.status("VerificationError: Found source_rev (r%s) with no corresponding target_rev: path_offset='%s'", source_rev_tmp, path_offset, color='RED')
308 error_cnt +=1
309 continue
310 target_revs_rmndr.remove(target_rev_tmp)
311 if target_revs_rmndr:
312 rmndr_list = ", ".join(map(str, target_revs_rmndr))
313 ui.status(" (%s/%s) Verify path: FAIL: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA, color='RED')
314 ui.status("VerificationError: Found one or more *extra* target_revs: path_offset='%s', target_revs='%s'", path_offset, rmndr_list, color='RED')
315 error_cnt +=1
316 else:
317 ui.status(" (%s/%s) Verify path: OK: %s", str(count).rjust(len(str(count_total))), count_total, path_offset, level=ui.EXTRA)
318
319 # Ensure there are no "extra" files in the target side
320 if options.verify == 2:
321 target_paths = []
322 child_paths = svnclient.list(target_url, target_rev, recursive=True)
323 for p in child_paths:
324 child_path_is_dir = True if p['kind'] == 'dir' else False
325 child_path_offset = p['path']
326 if not child_path_is_dir:
327 target_paths.append(child_path_offset)
328 # Compare
329 for path_offset in target_paths:
330 if not path_offset in check_paths:
331 ui.status("VerificationError: Path exists in target (@%s) but not source (@%s): %s", target_rev, source_rev, path_offset, color='RED')
332 error_cnt += 1
333 for path_offset in check_paths:
334 if not path_offset in target_paths:
335 ui.status("VerificationError: Path exists in source (@%s) but not target (@%s): %s", source_rev, target_rev, path_offset, color='RED')
336 error_cnt += 1
337
338 if error_cnt > 0:
339 raise VerificationError("Found %s verification errors" % (error_cnt))
340 ui.status("Verified revision %s (%s).", target_rev, "all" if options.verify == 2 else "only-changed")
341
342 def full_svn_revert():
343 """
344 Do an "svn revert" and proactively remove any extra files in the working copy.
345 """
346 run_svn(["revert", "--recursive", "."])
347 output = run_svn(["status"])
348 if output:
349 output_lines = output.strip("\n").split("\n")
350 for line in output_lines:
351 if line[0] == "?":
352 path = line[4:].strip(" ")
353 if os.path.isfile(path):
354 os.remove(path)
355 if os.path.isdir(path):
356 shell.rmtree(path)
357
358 def gen_tracking_revprops(source_rev):
359 """
360 Build an array of svn2svn-specific source-tracking revprops.
361 """
362 revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
363 {'name':'svn2svn:source_url', 'value':urllib.quote(source_url, ":/")},
364 {'name':'svn2svn:source_rev', 'value':source_rev}]
365 return revprops
366
367 def sync_svn_props(source_url, source_rev, path_offset):
368 """
369 Carry-forward any unversioned properties from the source repo to the
370 target WC.
371 """
372 source_props = svnclient.propget_all(join_path(source_url, path_offset), source_rev)
373 target_props = svnclient.propget_all(path_offset)
374 if 'svn:mergeinfo' in source_props:
375 # Never carry-forward "svn:mergeinfo"
376 del source_props['svn:mergeinfo']
377 for prop in target_props:
378 if prop not in source_props:
379 # Remove any properties which exist in target but not source
380 run_svn(["propdel", prop, svnclient.safe_path(path_offset)])
381 for prop in source_props:
382 if prop not in target_props or \
383 source_props[prop] != target_props[prop]:
384 # Set/update any properties which exist in source but not target or
385 # whose value differs between source vs. target.
386 run_svn(["propset", prop, source_props[prop], svnclient.safe_path(path_offset)])
387
388 def in_svn(p, require_in_repo=False, prefix=""):
389 """
390 Check if a given file/folder is being tracked by Subversion.
391 Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
392 With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
393 Use "svn status" to check the status of the file/folder.
394 """
395 entries = svnclient.status(p, non_recursive=True)
396 if not entries:
397 return False
398 d = entries[0]
399 if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
400 # If caller requires this path to be in the SVN repo, prevent returning True
401 # for paths that are only locally-added.
402 ret = False
403 else:
404 # Don't consider files tracked as deleted in the WC as under source-control.
405 # Consider files which are locally added/copied as under source-control.
406 ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
407 ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
408 return ret
409
410 def is_child_path(path, p_path):
411 return True if (path == p_path) or (path.startswith(p_path+"/")) else False
412
413 def join_path(base, child):
414 base.rstrip('/')
415 return base+"/"+child if child else base
416
417 def find_svn_ancestors(svn_repos_url, start_path, start_rev, stop_base_path=None, prefix=""):
418 """
419 Given an initial starting path+rev, walk the SVN history backwards to inspect the
420 ancestry of that path, optionally seeing if it traces back to stop_base_path.
421
422 Build an array of copyfrom_path and copyfrom_revision pairs for each of the "svn copy"'s.
423 If we find a copyfrom_path which stop_base_path is a substring match of (e.g. we crawled
424 back to the initial branch-copy from trunk), then return the collection of ancestor
425 paths. Otherwise, copyfrom_path has no ancestry compared to stop_base_path.
426
427 This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
428 file/folder was renamed in a branch and then that branch was merged back to trunk.
429
430 'svn_repos_url' is the full URL to the root of the SVN repository,
431 e.g. 'file:///path/to/repo'
432 'start_path' is the path in the SVN repo to the source path to start checking
433 ancestry at, e.g. '/branches/fix1/projectA/file1.txt'.
434 'start_rev' is the revision to start walking the history of start_path backwards from.
435 'stop_base_path' is the path in the SVN repo to stop tracing ancestry once we've reached,
436 i.e. the target path we're trying to trace ancestry back to, e.g. '/trunk'.
437 """
438 ui.status(prefix + ">> find_svn_ancestors: Start: (%s) start_path: %s stop_base_path: %s",
439 svn_repos_url, start_path+"@"+str(start_rev), stop_base_path, level=ui.DEBUG, color='YELLOW')
440 done = False
441 no_ancestry = False
442 cur_path = start_path
443 cur_rev = start_rev
444 first_iter_done = False
445 ancestors = []
446 while not done:
447 # Get the first "svn log" entry for cur_path (relative to @cur_rev)
448 ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url+cur_path+"@"+str(cur_rev), level=ui.DEBUG, color='YELLOW')
449 log_entry = svnclient.get_first_svn_log_entry(svn_repos_url+cur_path, 1, cur_rev)
450 if not log_entry:
451 ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
452 done = True
453 break
454 # If we found a copy-from case which matches our stop_base_path, we're done.
455 # ...but only if we've at least tried to search for the first copy-from path.
456 if stop_base_path is not None and first_iter_done and is_child_path(cur_path, stop_base_path):
457 ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(cur_path, stop_base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
458 done = True
459 break
460 first_iter_done = True
461 # Search for any actions on our target path (or parent paths).
462 changed_paths_temp = []
463 for d in log_entry['changed_paths']:
464 path = d['path']
465 if is_child_path(cur_path, path):
466 changed_paths_temp.append({'path': path, 'data': d})
467 if not changed_paths_temp:
468 # If no matches, then we've hit the end of the ancestry-chain.
469 ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
470 done = True
471 continue
472 # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
473 changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
474 # Find the action for our cur_path in this revision. Use a loop to check in reverse order,
475 # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from
476 # then we still correctly match the deepest copy-from.
477 for v in changed_paths:
478 d = v['data']
479 path = d['path']
480 # Check action-type for this file
481 action = d['action']
482 if action not in _valid_svn_actions:
483 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
484 % (log_entry['revision'], action))
485 ui.status(prefix + "> %s %s%s", action, path,
486 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
487 level=ui.DEBUG, color='YELLOW')
488 if action == 'D':
489 # If file/folder was deleted, ancestry-chain stops here
490 if stop_base_path:
491 no_ancestry = True
492 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
493 done = True
494 break
495 if action in 'RA':
496 # If file/folder was added/replaced but not a copy, ancestry-chain stops here
497 if not d['copyfrom_path']:
498 if stop_base_path:
499 no_ancestry = True
500 ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
501 "Added" if action == "A" else "Replaced",
502 level=ui.DEBUG, color='YELLOW')
503 done = True
504 break
505 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
506 # and keep checking for ancestors
507 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
508 action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
509 level=ui.DEBUG, color='YELLOW')
510 ancestors.append({'path': cur_path, 'revision': log_entry['revision'],
511 'copyfrom_path': cur_path.replace(d['path'], d['copyfrom_path']), 'copyfrom_rev': d['copyfrom_revision']})
512 cur_path = cur_path.replace(d['path'], d['copyfrom_path'])
513 cur_rev = d['copyfrom_revision']
514 # Follow the copy and keep on searching
515 break
516 if stop_base_path and no_ancestry:
517 # If we're tracing back ancestry to a specific target stop_base_path and
518 # the ancestry-chain stopped before we reached stop_base_path, then return
519 # nothing since there is no ancestry chaining back to that target.
520 ancestors = []
521 if ancestors:
522 if ui.get_level() >= ui.DEBUG:
523 max_len = 0
524 for idx in range(len(ancestors)):
525 d = ancestors[idx]
526 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
527 ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
528 for idx in range(len(ancestors)):
529 d = ancestors[idx]
530 ui.status(prefix + " [%s] %s --> %s", idx,
531 str(d['path']+"@"+str(d['revision'])).ljust(max_len),
532 str(d['copyfrom_path']+"@"+str(d['copyfrom_rev'])),
533 level=ui.DEBUG, color='YELLOW')
534 else:
535 ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
536 svn_repos_url+start_path+"@"+str(start_rev), level=ui.DEBUG, color='YELLOW')
537 return ancestors
538
539 def get_rev_map(source_rev, prefix):
540 """
541 Find the equivalent rev # in the target repo for the given rev # from the source repo.
542 """
543 ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
544 # Find the highest entry less-than-or-equal-to source_rev
545 for rev in range(int(source_rev), 0, -1):
546 in_rev_map = True if rev in rev_map else False
547 ui.status(prefix + ">> get_rev_map: rev=%s in_rev_map=%s", rev, str(in_rev_map), level=ui.DEBUG, color='BLACK_B')
548 if in_rev_map:
549 return int(rev_map[rev])
550 # Else, we fell off the bottom of the rev_map. Ruh-roh...
551 return None
552
553 def set_rev_map(source_rev, target_rev):
554 #ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
555 global rev_map
556 rev_map[int(source_rev)]=int(target_rev)
557
558 def build_rev_map(target_url, target_end_rev, source_info):
559 """
560 Check for any already-replayed history from source_url (source_info) and
561 build the mapping-table of source_rev -> target_rev.
562 """
563 global rev_map
564 rev_map = {}
565 ui.status("Rebuilding target_rev -> source_rev rev_map...", level=ui.VERBOSE)
566 proc_count = 0
567 it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
568 for log_entry in it_log_entries:
569 if log_entry['revprops']:
570 revprops = {}
571 for v in log_entry['revprops']:
572 if v['name'].startswith('svn2svn:'):
573 revprops[v['name']] = v['value']
574 if revprops and \
575 revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
576 revprops['svn2svn:source_url'] == urllib.quote(source_info['url'], ":/"):
577 source_rev = revprops['svn2svn:source_rev']
578 target_rev = log_entry['revision']
579 set_rev_map(source_rev, target_rev)
580 proc_count += 1
581 if proc_count % 500 == 0:
582 ui.status("...processed %s (%s of %s)..." % (proc_count, target_rev, target_end_rev), level=ui.VERBOSE)
583
584 def path_in_list(paths, path):
585 for p in paths:
586 if is_child_path(path, p):
587 return True
588 return False
589
590 def add_path(paths, path):
591 if not path_in_list(paths, path):
592 paths.append(path)
593
594 def in_ancestors(ancestors, ancestor):
595 match = True
596 for idx in range(len(ancestors)-1, 0, -1):
597 if int(ancestors[idx]['revision']) > ancestor['revision']:
598 match = is_child_path(ancestor['path'], ancestors[idx]['path'])
599 break
600 return match
601
602 def do_svn_add(source_url, path_offset, source_rev, source_ancestors, \
603 parent_copyfrom_path="", parent_copyfrom_rev="", \
604 export_paths={}, is_dir = False, skip_paths=[], prefix = ""):
605 """
606 Given the add'd source path, replay the "svn add/copy" commands to correctly
607 track renames across copy-from's.
608
609 For example, consider a sequence of events like this:
610 1. svn copy /trunk /branches/fix1
611 2. (Make some changes on /branches/fix1)
612 3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2 " Rename folder
613 4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt " Rename file inside renamed folder
614 5. svn co /trunk && svn merge /branches/fix1
615 After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
616 and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
617 to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
618 we'd lose the logical history that Proj2/file2.txt is really a descendant
619 of Proj1/file1.txt.
620
621 'path_offset' is the offset from source_base to the file to check ancestry for,
622 e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
623 'source_rev' is the revision ("svn log") that we're processing from the source repo.
624 'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
625 directory, when being called recursively by do_svn_add_dir().
626 'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
627 'is_dir' is whether path_offset is a directory (rather than a file).
628 """
629 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
630 ui.status(prefix + ">> do_svn_add: %s %s", join_path(source_base, path_offset)+"@"+str(source_rev),
631 " (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
632 level=ui.DEBUG, color='GREEN')
633 # Check if the given path has ancestors which chain back to the current source_base
634 found_ancestor = False
635 ancestors = find_svn_ancestors(source_repos_url, join_path(source_base, path_offset), source_rev, stop_base_path=source_base, prefix=prefix+" ")
636 ancestor = ancestors[len(ancestors)-1] if ancestors else None # Choose the eldest ancestor, i.e. where we reached stop_base_path=source_base
637 if ancestor and not in_ancestors(source_ancestors, ancestor):
638 ancestor = None
639 copyfrom_path = ancestor['copyfrom_path'] if ancestor else ""
640 copyfrom_rev = ancestor['copyfrom_rev'] if ancestor else ""
641 if ancestor:
642 # The copy-from path has ancestry back to source_url.
643 ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
644 level=ui.DEBUG, color='GREEN', bold=True)
645 found_ancestor = True
646 # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
647 # return None in the case where copyfrom_rev is *before* our source_start_rev.
648 tgt_rev = get_rev_map(copyfrom_rev, prefix+" ")
649 ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
650 else:
651 ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
652 found_ancestor = False
653 if found_ancestor and tgt_rev:
654 # Check if this path_offset in the target WC already has this ancestry, in which
655 # case there's no need to run the "svn copy" (again).
656 path_in_svn = in_svn(path_offset, prefix=prefix+" ")
657 log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+" ") else []
658 if (not log_entry or (log_entry['revision'] != tgt_rev)):
659 copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
660 ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
661 ui.status(prefix + " copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
662 ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
663 if path_in_svn and \
664 ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
665 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
666 # When being called recursively, if this child entry has the same ancestor as the
667 # the parent, then no need to try to run another "svn copy".
668 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
669 parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
670 pass
671 else:
672 # Copy this path from the equivalent path+rev in the target repo, to create the
673 # equivalent history.
674 if parent_copyfrom_path:
675 # If we have a parent copy-from path, we mis-match that so display a status
676 # message describing the action we're mimic'ing. If path_in_svn, then this
677 # is logically a "replace" rather than an "add".
678 ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), join_path(source_base, path_offset), ancestors[0]['copyfrom_path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
679 if path_in_svn:
680 # If local file is already under version-control, then this is a replace.
681 ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
682 svnclient.update(path_offset)
683 svnclient.remove(path_offset, force=True)
684 run_svn(["copy", "-r", tgt_rev, svnclient.safe_path(join_path(target_url, copyfrom_offset), tgt_rev), svnclient.safe_path(path_offset)])
685 if is_dir:
686 # Export the final verison of all files in this folder.
687 add_path(export_paths, path_offset)
688 else:
689 # Export the final verison of this file.
690 svnclient.export(source_repos_url+join_path(source_base, path_offset), source_rev, path_offset, force=True)
691 if options.keep_prop:
692 sync_svn_props(source_url, source_rev, path_offset)
693 else:
694 ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
695 else:
696 # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
697 # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
698 # Create (parent) directory if needed.
699 # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
700 # split-out to a shared tag?
701 p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or None
702 if p_path and not os.path.exists(p_path):
703 run_svn(["mkdir", svnclient.safe_path(p_path)])
704 if not in_svn(path_offset, prefix=prefix+" "):
705 if is_dir:
706 # Export the final verison of all files in this folder.
707 add_path(export_paths, path_offset)
708 else:
709 # Export the final verison of this file. We *need* to do this before running
710 # the "svn add", even if we end-up re-exporting this file again via export_paths.
711 svnclient.export(source_repos_url+join_path(source_base, path_offset), source_rev, path_offset, force=True)
712 # If not already under version-control, then "svn add" this file/folder.
713 run_svn(["add", "--parents", svnclient.safe_path(path_offset)])
714 if options.keep_prop:
715 sync_svn_props(source_url, source_rev, path_offset)
716 if is_dir:
717 # For any folders that we process, process any child contents, so that we correctly
718 # replay copies/replaces/etc.
719 do_svn_add_dir(source_url, path_offset, source_rev, source_ancestors,
720 copyfrom_path, copyfrom_rev, export_paths, skip_paths, prefix+" ")
721
722 def do_svn_add_dir(source_url, path_offset, source_rev, source_ancestors, \
723 parent_copyfrom_path, parent_copyfrom_rev, \
724 export_paths, skip_paths, prefix=""):
725 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
726 # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
727 # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
728 # associated remote repo folder. (Is this a problem?)
729 paths_local = svnclient.list(path_offset)
730 paths_remote = svnclient.list(join_path(source_url, path_offset), source_rev)
731 ui.status(prefix + ">> do_svn_add_dir: paths_local: %s", str(paths_local), level=ui.DEBUG, color='GREEN')
732 ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
733 # Update files/folders which exist in remote but not local
734 for p in paths_remote:
735 path_is_dir = True if p['kind'] == 'dir' else False
736 working_path = join_path(path_offset, p['path']).lstrip('/')
737 #print "working_path:%s = path_offset:%s + path:%s" % (working_path, path_offset, path)
738 if not working_path in skip_paths:
739 do_svn_add(source_url, working_path, source_rev, source_ancestors,
740 parent_copyfrom_path, parent_copyfrom_rev,
741 export_paths, path_is_dir, skip_paths, prefix+" ")
742 # Remove files/folders which exist in local but not remote
743 for p in paths_local:
744 if not p in paths_remote:
745 working_path = join_path(path_offset, p['path']).lstrip('/')
746 ui.status(" %s %s", 'D', join_path(source_base, working_path), level=ui.VERBOSE)
747 svnclient.update(working_path)
748 svnclient.remove(working_path, force=True)
749 # TODO: Does this handle deleted folders too? Wouldn't want to have a case
750 # where we only delete all files from folder but leave orphaned folder around.
751
752 def process_svn_log_entry(log_entry, ancestors, commit_paths, prefix = ""):
753 """
754 Process SVN changes from the given log entry. Build an array (commit_paths)
755 of the paths in the working-copy that were changed, i.e. the paths which
756 we'll pass to "svn commit".
757 """
758 export_paths = []
759 source_rev = log_entry['revision']
760 source_url = log_entry['url']
761 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
762 ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
763 for d in log_entry['changed_paths']:
764 # Get the full path for this changed_path
765 # e.g. '/branches/bug123/projectA/file1.txt'
766 path = d['path']
767 if not is_child_path(path, source_base):
768 # Ignore changed files that are not part of this subdir
769 ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
770 continue
771 if d['kind'] == "" or d['kind'] == 'none':
772 # The "kind" value was introduced in SVN 1.6, and "svn log --xml" won't return a "kind"
773 # value for commits made on a pre-1.6 repo, even if the server is now running 1.6.
774 # We need to use other methods to fetch the node-kind for these cases.
775 d['kind'] = svnclient.get_kind(source_repos_url, path, source_rev, d['action'], log_entry['changed_paths'])
776 assert (d['kind'] == 'file') or (d['kind'] == 'dir')
777 path_is_dir = True if d['kind'] == 'dir' else False
778 path_is_file = True if d['kind'] == 'file' else False
779 # Calculate the offset (based on source_base) for this changed_path
780 # e.g. 'projectA/file1.txt'
781 # (path = source_base + "/" + path_offset)
782 path_offset = path[len(source_base):].strip("/")
783 # Get the action for this path
784 action = d['action']
785 if action not in _valid_svn_actions:
786 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
787 % (source_rev, action))
788 ui.status(" %s %s%s", action, d['path'],
789 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
790 level=ui.VERBOSE)
791
792 # Try to be efficient and keep track of an explicit list of paths in the
793 # working copy that changed. If we commit from the root of the working copy,
794 # then SVN needs to crawl the entire working copy looking for pending changes.
795 commit_paths.append(path_offset)
796
797 # Special-handling for replace's
798 if action == 'R':
799 # If file was "replaced" (deleted then re-added, all in same revision),
800 # then we need to run the "svn rm" first, then change action='A'. This
801 # lets the normal code below handle re-"svn add"'ing the files. This
802 # should replicate the "replace".
803 if path_offset and in_svn(path_offset):
804 # Target path might not be under version-control yet, e.g. parent "add"
805 # was a copy-from a branch which had no ancestry back to trunk, and each
806 # child folder under that parent folder is a "replace" action on the final
807 # merge to trunk. Since the child folders will be in skip_paths, do_svn_add
808 # wouldn't have created them while processing the parent "add" path.
809 if path_is_dir:
810 # Need to "svn update" before "svn remove" in case child contents are at
811 # a higher rev than the (parent) path_offset.
812 svnclient.update(path_offset)
813 svnclient.remove(path_offset, force=True)
814 action = 'A'
815
816 # Handle all the various action-types
817 # (Handle "add" first, for "svn copy/move" support)
818 if action == 'A':
819 # Determine where to export from.
820 svn_copy = False
821 # Handle cases where this "add" was a copy from another URL in the source repo
822 if d['copyfrom_revision']:
823 copyfrom_path = d['copyfrom_path']
824 copyfrom_rev = d['copyfrom_revision']
825 skip_paths = []
826 for tmp_d in log_entry['changed_paths']:
827 tmp_path = tmp_d['path']
828 if is_child_path(tmp_path, path) and tmp_d['action'] in 'ARD':
829 # Build list of child entries which are also in the changed_paths list,
830 # so that do_svn_add() can skip processing these entries when recursing
831 # since we'll end-up processing them later. Don't include action="M" paths
832 # in this list because it's non-conclusive: it could just mean that the
833 # file was modified *after* the copy-from, so we still want do_svn_add()
834 # to re-create the correct ancestry.
835 tmp_path_offset = tmp_path[len(source_base):].strip("/")
836 skip_paths.append(tmp_path_offset)
837 do_svn_add(source_url, path_offset, source_rev, ancestors, "", "", export_paths, path_is_dir, skip_paths, prefix+" ")
838 # Else just "svn export" the files from the source repo and "svn add" them.
839 else:
840 # Create (parent) directory if needed
841 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or None
842 if p_path and not os.path.exists(p_path):
843 run_svn(["mkdir", svnclient.safe_path(p_path)])
844 # Export the entire added tree.
845 if path_is_dir:
846 # For directories, defer the (recurisve) "svn export". Might have a
847 # situation in a branch merge where the entry in the svn-log is a
848 # non-copy-from'd "add" but there are child contents (that we haven't
849 # gotten to yet in log_entry) that are copy-from's. When we try do
850 # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
851 # having pre-existing (svn-add'd) contents creates some trouble.
852 # Instead, just create the stub folders ("svn mkdir" above) and defer
853 # exporting the final file-state until the end.
854 add_path(export_paths, path_offset)
855 else:
856 # Export the final verison of this file. We *need* to do this before running
857 # the "svn add", even if we end-up re-exporting this file again via export_paths.
858 svnclient.export(join_path(source_url, path_offset), source_rev, path_offset, force=True)
859 if not in_svn(path_offset, prefix=prefix+" "):
860 # Need to use in_svn here to handle cases where client committed the parent
861 # folder and each indiv sub-folder.
862 run_svn(["add", "--parents", svnclient.safe_path(path_offset)])
863 if options.keep_prop:
864 sync_svn_props(source_url, source_rev, path_offset)
865
866 elif action == 'D':
867 if path_is_dir:
868 # For dirs, need to "svn update" before "svn remove" because the final
869 # "svn commit" will fail if the parent (path_offset) is at a lower rev
870 # than any of the child contents. This needs to be a recursive update.
871 svnclient.update(path_offset)
872 svnclient.remove(path_offset, force=True)
873
874 elif action == 'M':
875 if path_is_file:
876 svnclient.export(join_path(source_url, path_offset), source_rev, path_offset, force=True, non_recursive=True)
877 if path_is_dir:
878 # For dirs, need to "svn update" before export/prop-sync because the
879 # final "svn commit" will fail if the parent is at a lower rev than
880 # child contents. Just need to update the rev-state of the dir (d['path']),
881 # don't need to recursively update all child contents.
882 # (??? is this the right reason?)
883 svnclient.update(path_offset, non_recursive=True)
884 if options.keep_prop:
885 sync_svn_props(source_url, source_rev, path_offset)
886
887 else:
888 raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
889 % action)
890
891 # Export the final version of all add'd paths from source_url
892 if export_paths:
893 for path_offset in export_paths:
894 svnclient.export(join_path(source_url, path_offset), source_rev, path_offset, force=True)
895
896 def keep_revnum(source_rev, target_rev_last, wc_target_tmp):
897 """
898 Add "padding" target revisions as needed to keep source and target
899 revision #'s identical.
900 """
901 bh = BreakHandler()
902 if int(source_rev) <= int(target_rev_last):
903 raise InternalError("keep-revnum mode is enabled, "
904 "but source revision (r%s) is less-than-or-equal last target revision (r%s)" % \
905 (source_rev, target_rev_last))
906 if int(target_rev_last) < int(source_rev)-1:
907 # Add "padding" target revisions to keep source and target rev #'s identical
908 if os.path.exists(wc_target_tmp):
909 shell.rmtree(wc_target_tmp)
910 run_svn(["checkout", "-r", "HEAD", "--depth=empty", svnclient.safe_path(target_repos_url, "HEAD"), svnclient.safe_path(wc_target_tmp)])
911 for rev_num in range(int(target_rev_last)+1, int(source_rev)):
912 run_svn(["propset", "svn2svn:keep-revnum", rev_num, svnclient.safe_path(wc_target_tmp)])
913 # Prevent Ctrl-C's during this inner part, so we'll always display
914 # the "Commit revision ..." message if we ran a "svn commit".
915 bh.enable()
916 output = run_svn(["commit", "-m", "", svnclient.safe_path(wc_target_tmp)])
917 rev_num_tmp = parse_svn_commit_rev(output) if output else None
918 assert rev_num == rev_num_tmp
919 ui.status("Committed revision %s (keep-revnum).", rev_num)
920 bh.disable()
921 # Check if the user tried to press Ctrl-C
922 if bh.trapped:
923 raise KeyboardInterrupt
924 target_rev_last = rev_num
925 shell.rmtree(wc_target_tmp)
926 return target_rev_last
927
928 def disp_svn_log_summary(log_entry):
929 ui.status("------------------------------------------------------------------------", level=ui.VERBOSE)
930 ui.status("r%s | %s | %s",
931 log_entry['revision'],
932 log_entry['author'],
933 str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')), level=ui.VERBOSE)
934 ui.status(log_entry['message'], level=ui.VERBOSE)
935
936 def real_main(args):
937 global source_url, target_url, rev_map
938 # Use urllib.unquote() to URL-decode source_url/target_url values.
939 # All URLs passed to run_svn() should go through svnclient.safe_path()
940 # and we don't want to end-up *double* urllib.quote'ing if the user-
941 # supplied source/target URL's are already URL-encoded.
942 source_url = urllib.unquote(args.pop(0).rstrip("/")) # e.g. 'http://server/svn/source/trunk'
943 target_url = urllib.unquote(args.pop(0).rstrip("/")) # e.g. 'file:///svn/target/trunk'
944 ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
945
946 # Make sure that both the source and target URL's are valid
947 source_info = svnclient.info(source_url)
948 assert is_child_path(source_url, source_info['repos_url'])
949 target_info = svnclient.info(target_url)
950 assert is_child_path(target_url, target_info['repos_url'])
951
952 # Init global vars
953 global source_repos_url,source_base,source_repos_uuid
954 source_repos_url = source_info['repos_url'] # e.g. 'http://server/svn/source'
955 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
956 source_repos_uuid = source_info['repos_uuid']
957 global target_repos_url,target_base
958 target_repos_url = target_info['repos_url'] # e.g. 'http://server/svn/target'
959 target_base = target_url[len(target_repos_url):] # e.g. '/trunk'
960
961 # Init start and end revision
962 try:
963 source_start_rev = svnclient.get_rev(source_repos_url, options.rev_start if options.rev_start else 1)
964 except ExternalCommandFailed:
965 print "Error: Invalid start source revision value: %s" % (options.rev_start)
966 sys.exit(1)
967 try:
968 source_end_rev = svnclient.get_rev(source_repos_url, options.rev_end if options.rev_end else "HEAD")
969 except ExternalCommandFailed:
970 print "Error: Invalid end source revision value: %s" % (options.rev_end)
971 sys.exit(1)
972 ui.status("Using source revision range %s:%s", source_start_rev, source_end_rev, level=ui.VERBOSE)
973
974 # TODO: If options.keep_date, should we try doing a "svn propset" on an *existing* revision
975 # as a sanity check, so we check if the pre-revprop-change hook script is correctly setup
976 # before doing first replay-commit?
977
978 target_rev_last = target_info['revision'] # Last revision # in the target repo
979 wc_target = os.path.abspath('_wc_target')
980 wc_target_tmp = os.path.abspath('_wc_target_tmp')
981 num_entries_proc = 0
982 commit_count = 0
983 source_rev = None
984 target_rev = None
985
986 # Check out a working copy of target_url if needed
987 wc_exists = os.path.exists(wc_target)
988 if wc_exists and not options.cont_from_break:
989 shell.rmtree(wc_target)
990 wc_exists = False
991 if not wc_exists:
992 ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
993 svnclient.svn_checkout(target_url, wc_target)
994 os.chdir(wc_target)
995 if wc_exists:
996 # If using an existing WC, make sure it's clean ("svn revert")
997 ui.status("Cleaning-up _wc_target...", level=ui.VERBOSE)
998 run_svn(["cleanup"])
999 full_svn_revert()
1000
1001 if not options.cont_from_break:
1002 # Warn user if trying to start (non-continue) into a non-empty target path
1003 if not options.force_nocont:
1004 top_paths = svnclient.list(target_url, "HEAD")
1005 if len(top_paths)>0:
1006 print "Error: Trying to replay (non-continue-mode) into a non-empty target_url location. " \
1007 "Use --force if you're sure this is what you want."
1008 sys.exit(1)
1009 # Get the first log entry at/after source_start_rev, which is where
1010 # we'll do the initial import from.
1011 source_ancestors = find_svn_ancestors(source_repos_url, source_base, source_end_rev, prefix=" ")
1012 it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False, ancestors=source_ancestors)
1013 source_start_log = None
1014 for log_entry in it_log_start:
1015 # Pick the first entry. Need to use a "for ..." loop since we're using an iterator.
1016 source_start_log = log_entry
1017 break
1018 if not source_start_log:
1019 raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
1020 (source_start_rev, source_end_rev, source_url))
1021
1022 # This is the revision we will start from for source_url
1023 source_start_rev = int(source_start_log['revision'])
1024 ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
1025 ui.status("", level=ui.VERBOSE)
1026 if options.keep_revnum and source_rev > target_rev_last:
1027 target_rev_last = keep_revnum(source_rev, target_rev_last, wc_target_tmp)
1028
1029 # For the initial commit to the target URL, export all the contents from
1030 # the source URL at the start-revision.
1031 disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_repos_url, source_start_rev, source_start_rev))
1032 # Export and add file-contents from source_url@source_start_rev
1033 source_start_url = source_url if not source_ancestors else source_repos_url+source_ancestors[len(source_ancestors)-1]['copyfrom_path']
1034 top_paths = svnclient.list(source_start_url, source_start_rev)
1035 for p in top_paths:
1036 # For each top-level file/folder...
1037 path_is_dir = True if p['kind'] == "dir" else False
1038 path_offset = p['path']
1039 if in_svn(path_offset, prefix=" "):
1040 raise InternalError("Cannot replay history on top of pre-existing structure: %s" % join_path(source_start_url, path_offset))
1041 if path_is_dir and not os.path.exists(path_offset):
1042 os.makedirs(path_offset)
1043 svnclient.export(join_path(source_start_url, path_offset), source_start_rev, path_offset, force=True)
1044 run_svn(["add", svnclient.safe_path(path_offset)])
1045 # Update any properties on the newly added content
1046 paths = svnclient.list(source_start_url, source_start_rev, recursive=True)
1047 if options.keep_prop:
1048 sync_svn_props(source_start_url, source_start_rev, "")
1049 for p in paths:
1050 path_offset = p['path']
1051 ui.status(" A %s", join_path(source_base, path_offset), level=ui.VERBOSE)
1052 if options.keep_prop:
1053 sync_svn_props(source_start_url, source_start_rev, path_offset)
1054 # Commit the initial import
1055 num_entries_proc += 1
1056 target_revprops = gen_tracking_revprops(source_start_rev) # Build source-tracking revprop's
1057 target_rev = commit_from_svn_log_entry(source_start_log, target_revprops=target_revprops)
1058 if target_rev:
1059 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
1060 set_rev_map(source_start_rev, target_rev)
1061 commit_count += 1
1062 target_rev_last = target_rev
1063 if options.verify:
1064 verify_commit(source_rev, target_rev_last)
1065 else:
1066 # Re-build the rev_map based on any already-replayed history in target_url
1067 build_rev_map(target_url, target_rev_last, source_info)
1068 if not rev_map:
1069 print "Error: Called with continue-mode, but no already-replayed source history found in target_url."
1070 sys.exit(1)
1071 source_start_rev = int(max(rev_map, key=rev_map.get))
1072 assert source_start_rev
1073 ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
1074 ui.status("", level=ui.VERBOSE)
1075
1076 svn_vers_t = svnclient.version()
1077 svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
1078
1079 # Load SVN log starting from source_start_rev + 1
1080 source_ancestors = find_svn_ancestors(source_repos_url, source_base, source_end_rev, prefix=" ")
1081 it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True, ancestors=source_ancestors) if source_start_rev < source_end_rev else []
1082 source_rev_last = source_start_rev
1083
1084 try:
1085 for log_entry in it_log_entries:
1086 if options.entries_proc_limit:
1087 if num_entries_proc >= options.entries_proc_limit:
1088 break
1089 # Replay this revision from source_url into target_url
1090 source_rev = log_entry['revision']
1091 log_url = log_entry['url']
1092 #print "source_url:%s log_url:%s" % (source_url, log_url)
1093 if options.keep_revnum:
1094 if source_rev < target_rev_last:
1095 print "Error: Last target revision (r%s) is equal-or-higher than starting source revision (r%s). " \
1096 "Cannot use --keep-revnum mode." % (target_rev_last, source_start_rev)
1097 sys.exit(1)
1098 target_rev_last = keep_revnum(source_rev, target_rev_last, wc_target_tmp)
1099 disp_svn_log_summary(log_entry)
1100 # Process all the changed-paths in this log entry
1101 commit_paths = []
1102 process_svn_log_entry(log_entry, source_ancestors, commit_paths)
1103 num_entries_proc += 1
1104 # Commit any changes made to _wc_target
1105 target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's
1106 target_rev = commit_from_svn_log_entry(log_entry, commit_paths, target_revprops=target_revprops)
1107 source_rev_last = source_rev
1108 if target_rev:
1109 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
1110 source_rev = log_entry['revision']
1111 set_rev_map(source_rev, target_rev)
1112 target_rev_last = target_rev
1113 commit_count += 1
1114 if options.verify:
1115 verify_commit(source_rev, target_rev_last, log_entry)
1116 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
1117 if svn_vers >= 1.7 and (commit_count % 100 == 0):
1118 run_svn(["cleanup"])
1119 if source_rev_last == source_start_rev:
1120 # If there were no new source_url revisions to process, still trigger
1121 # "full-mode" verify check (if enabled).
1122 if options.verify:
1123 verify_commit(source_rev_last, target_rev_last)
1124
1125 except KeyboardInterrupt:
1126 print "\nStopped by user."
1127 print "\nCleaning-up..."
1128 run_svn(["cleanup"])
1129 full_svn_revert()
1130 except:
1131 print "\nCommand failed with following error:\n"
1132 traceback.print_exc()
1133 print "\nCleaning-up..."
1134 run_svn(["cleanup"])
1135 print run_svn(["status"])
1136 full_svn_revert()
1137 finally:
1138 print "\nFinished at source revision %s%s." % (source_rev_last, " (dry-run)" if options.dry_run else "")
1139
1140 def main():
1141 # Defined as entry point. Must be callable without arguments.
1142 usage = "svn2svn, version %s\n" % str(full_version) + \
1143 "<http://nynim.org/projects/svn2svn> <https://github.com/tonyduckles/svn2svn>\n\n" + \
1144 "Usage: %prog [OPTIONS] source_url target_url\n"
1145 description = """\
1146 Replicate (replay) history from one SVN repository to another. Maintain
1147 logical ancestry wherever possible, so that 'svn log' on the replayed repo
1148 will correctly follow file/folder renames.
1149
1150 Examples:
1151 Create a copy of only /trunk from source repo, starting at r5000
1152 $ svnadmin create /svn/target
1153 $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
1154 $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
1155 1. The target_url will be checked-out to ./_wc_target
1156 2. The first commit to http://server/source/trunk at/after r5000 will be
1157 exported & added into _wc_target
1158 3. All revisions affecting http://server/source/trunk (starting at r5000)
1159 will be replayed to _wc_target. Any add/copy/move/replaces that are
1160 copy-from'd some path outside of /trunk (e.g. files renamed on a
1161 /branch and branch was merged into /trunk) will correctly maintain
1162 logical ancestry where possible.
1163
1164 Use continue-mode (-c) to pick-up where the last run left-off
1165 $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
1166 1. The target_url will be checked-out to ./_wc_target, if not already
1167 checked-out
1168 2. All new revisions affecting http://server/source/trunk starting from
1169 the last replayed revision to file:///svn/target/trunk (based on the
1170 svn2svn:* revprops) will be replayed to _wc_target, maintaining all
1171 logical ancestry where possible."""
1172 parser = optparse.OptionParser(usage, description=description,
1173 formatter=HelpFormatter(), version="%prog "+str(full_version))
1174 parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
1175 help="enable additional output (use -vv or -vvv for more)")
1176 parser.add_option("-a", "--archive", action="store_true", dest="archive", default=False,
1177 help="archive/mirror mode; same as -UDP (see REQUIRE's below)\n"
1178 "maintain same commit author, same commit time, and file/dir properties")
1179 parser.add_option("-U", "--keep-author", action="store_true", dest="keep_author", default=False,
1180 help="maintain same commit authors (svn:author) as source\n"
1181 "(REQUIRES 'pre-revprop-change' hook script to allow 'svn:author' changes)")
1182 parser.add_option("-D", "--keep-date", action="store_true", dest="keep_date", default=False,
1183 help="maintain same commit time (svn:date) as source\n"
1184 "(REQUIRES 'pre-revprop-change' hook script to allow 'svn:date' changes)")
1185 parser.add_option("-P", "--keep-prop", action="store_true", dest="keep_prop", default=False,
1186 help="maintain same file/dir SVN properties as source")
1187 parser.add_option("-R", "--keep-revnum", action="store_true", dest="keep_revnum", default=False,
1188 help="maintain same rev #'s as source. creates placeholder target "
1189 "revisions (by modifying a 'svn2svn:keep-revnum' property at the root of the target repo)")
1190 parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
1191 help="continue from last source commit to target (based on svn2svn:* revprops)")
1192 parser.add_option("-f", "--force", action="store_true", dest="force_nocont",
1193 help="allow replaying into a non-empty target folder")
1194 parser.add_option("-r", "--revision", type="string", dest="revision", metavar="ARG",
1195 help="revision range to replay from source_url\n"
1196 "A revision argument can be one of:\n"
1197 " START start rev # (end will be 'HEAD')\n"
1198 " START:END start and ending rev #'s\n"
1199 "Any revision # formats which SVN understands are "
1200 "supported, e.g. 'HEAD', '{2010-01-31}', etc.")
1201 parser.add_option("-u", "--log-author", action="store_true", dest="log_author", default=False,
1202 help="append source commit author to replayed commit mesages")
1203 parser.add_option("-d", "--log-date", action="store_true", dest="log_date", default=False,
1204 help="append source commit time to replayed commit messages")
1205 parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
1206 help="maximum number of source revisions to process")
1207 parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
1208 help="process next source revision but don't commit changes to "
1209 "target working-copy (forces --limit=1)")
1210 parser.add_option("-x", "--verify", action="store_const", const=1, dest="verify",
1211 help="verify ancestry and content for changed paths in commit after every target commit or last target commit")
1212 parser.add_option("-X", "--verify-all", action="store_const", const=2, dest="verify",
1213 help="verify ancestry and content for entire target_url tree after every target commit or last target commit")
1214 parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
1215 help="enable debugging output (same as -vvv)")
1216 global options
1217 options, args = parser.parse_args()
1218 if len(args) != 2:
1219 parser.error("incorrect number of arguments")
1220 if options.verbosity < 10:
1221 # Expand multiple "-v" arguments to a real ui._level value
1222 options.verbosity *= 10
1223 if options.dry_run:
1224 # When in dry-run mode, only try to process the next log_entry
1225 options.entries_proc_limit = 1
1226 options.rev_start = None
1227 options.rev_end = None
1228 if options.revision:
1229 # Reg-ex for matching a revision arg (http://svnbook.red-bean.com/en/1.5/svn.tour.revs.specifiers.html#svn.tour.revs.dates)
1230 rev_patt = '[0-9A-Z]+|\{[0-9A-Za-z/\\ :-]+\}'
1231 rev = None
1232 match = re.match('^('+rev_patt+'):('+rev_patt+')$', options.revision) # First try start:end match
1233 if match is None: match = re.match('^('+rev_patt+')$', options.revision) # Next, try start match
1234 if match is None:
1235 parser.error("unexpected --revision argument format; see 'svn help log' for valid revision formats")
1236 rev = match.groups()
1237 options.rev_start = rev[0] if len(rev)>0 else None
1238 options.rev_end = rev[1] if len(rev)>1 else None
1239 if options.archive:
1240 options.keep_author = True
1241 options.keep_date = True
1242 options.keep_prop = True
1243 ui.update_config(options)
1244 return real_main(args)
1245
1246
1247 if __name__ == "__main__":
1248 sys.exit(main() or 0)