]> Tony Duckles's Git Repositories (git.nynim.org) - svn2svn.git/blob - svn2svn/run/svn2svn.py
Remove HEAD-specific short-circuiting
[svn2svn.git] / svn2svn / run / svn2svn.py
1 """
2 Replicate (replay) changesets from one SVN repository to another:
3 * Maintains full logical history (e.g. uses "svn copy" for renames).
4 * Maintains original commit messages.
5 * Optionally maintain source author info. (Only supported if accessing
6 target SVN repo via file://)
7 * Cannot maintain original commit date, but appends original commit date
8 for each commit message: "Date: %d".
9 * Optionally run an external shell script before each replayed commit
10 to give the ability to dynamically exclude or modify files as part
11 of the replay.
12
13 License: GPLv3, same as hgsvn (https://bitbucket.org/andialbrecht/hgsvn)
14 Author: Tony Duckles (https://github.com/tonyduckles/svn2svn)
15 (Inspired by http://code.google.com/p/svn2svn/, and uses code for hgsvn
16 for SVN client handling)
17 """
18
19 from .. import base_version, full_version
20 from .. import ui
21 from .. import svnclient
22 from ..shell import run_svn
23 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction)
24
25 import sys
26 import os
27 import time
28 import traceback
29 from optparse import OptionParser,OptionGroup
30 from datetime import datetime
31 from operator import itemgetter
32
33 def commit_from_svn_log_entry(entry, files=None, keep_author=False, revprops=[]):
34 """
35 Given an SVN log entry and an optional sequence of files, do an svn commit.
36 """
37 # TODO: Run optional external shell hook here, for doing pre-commit filtering
38 # This will use the local timezone for displaying commit times
39 timestamp = int(entry['date'])
40 svn_date = str(datetime.fromtimestamp(timestamp))
41 # Uncomment this one one if you prefer UTC commit times
42 #svn_date = "%d 0" % timestamp
43 if keep_author:
44 options = ["commit", "--force-log", "-m", entry['message'] + "\nDate: " + svn_date, "--username", entry['author']]
45 else:
46 options = ["commit", "--force-log", "-m", entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + entry['author']]
47 if revprops:
48 for r in revprops:
49 options += ["--with-revprop", r['name']+"="+str(r['value'])]
50 if files:
51 options += list(files)
52 output = run_svn(options)
53 if output:
54 output_lines = output.strip("\n").split("\n")
55 rev = ""
56 for line in output_lines:
57 if line[0:19] == 'Committed revision ':
58 rev = line[19:].rstrip('.')
59 break
60 if rev:
61 ui.status("Committed revision %s.", rev)
62
63 def in_svn(p, require_in_repo=False, prefix=""):
64 """
65 Check if a given file/folder is being tracked by Subversion.
66 Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
67 With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
68 Use "svn status" to check the status of the file/folder.
69 """
70 entries = svnclient.get_svn_status(p, no_recursive=True)
71 if not entries:
72 return False
73 d = entries[0]
74 if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
75 # If caller requires this path to be in the SVN repo, prevent returning True
76 # for paths that are only locally-added.
77 ret = False
78 else:
79 # Don't consider files tracked as deleted in the WC as under source-control.
80 # Consider files which are locally added/copied as under source-control.
81 ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
82 ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
83 return ret
84
85 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
86 """
87 Given a source path, walk the SVN history backwards to inspect the ancestory of
88 that path, seeing if it traces back to base_path. Build an array of copyfrom_path
89 and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
90 which base_path is a substring match of (e.g. we crawled back to the initial branch-
91 copy from trunk), then return the collection of ancestor paths. Otherwise,
92 copyfrom_path has no ancestory compared to base_path.
93
94 This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
95 file/folder was renamed in a branch and then that branch was merged back to trunk.
96
97 'svn_repos_url' is the full URL to the root of the SVN repository,
98 e.g. 'file:///path/to/repo'
99 'base_path' is the path in the SVN repo to the target path we're trying to
100 trace ancestry back to, e.g. 'trunk'.
101 'source_path' is the path in the SVN repo to the source path to start checking
102 ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
103 (full_path = svn_repos_url+base_path+"/"+path_offset)
104 'source_rev' is the revision to start walking the history of source_path backwards from.
105 """
106 ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s base_path: %s",
107 svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
108 done = False
109 working_path = base_path+"/"+source_path
110 working_rev = source_rev
111 first_iter_done = False
112 ancestors_temp = []
113 while not done:
114 # Get the first "svn log" entry for this path (relative to @rev)
115 ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
116 log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path+"@"+str(working_rev), 1, working_rev, True)
117 if not log_entry:
118 ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
119 done = True
120 break
121 # If we found a copy-from case which matches our base_path, we're done.
122 # ...but only if we've at least tried to search for the first copy-from path.
123 if first_iter_done and working_path.startswith(base_path):
124 ui.status(prefix + ">> find_svn_ancestors: Done: Found working_path.startswith(base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
125 done = True
126 break
127 first_iter_done = True
128 # Search for any actions on our target path (or parent paths).
129 changed_paths_temp = []
130 for d in log_entry['changed_paths']:
131 path = d['path']
132 if path in working_path:
133 changed_paths_temp.append({'path': path, 'data': d})
134 if not changed_paths_temp:
135 # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
136 ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
137 done = True
138 continue
139 # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
140 changed_paths = sorted(changed_paths_temp, key=itemgetter('path'), reverse=True)
141 # Find the action for our working_path in this revision. Use a loop to check in reverse order,
142 # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
143 for v in changed_paths:
144 d = v['data']
145 path = d['path']
146 # Check action-type for this file
147 action = d['action']
148 if action not in 'MARD':
149 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
150 % (log_entry['revision'], action))
151 ui.status(prefix + "> %s %s%s", action, path,
152 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
153 level=ui.DEBUG, color='YELLOW')
154 if action == 'D':
155 # If file/folder was deleted, it has no ancestor
156 ancestors_temp = []
157 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
158 done = True
159 break
160 if action in 'RA':
161 # If file/folder was added/replaced but not a copy, it has no ancestor
162 if not d['copyfrom_path']:
163 ancestors_temp = []
164 ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
165 "Added" if action == "A" else "Replaced",
166 level=ui.DEBUG, color='YELLOW')
167 done = True
168 break
169 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
170 # and keep checking for ancestors
171 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
172 action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
173 level=ui.DEBUG, color='YELLOW')
174 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
175 'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
176 working_path = working_path.replace(d['path'], d['copyfrom_path'])
177 working_rev = d['copyfrom_revision']
178 # Follow the copy and keep on searching
179 break
180 ancestors = []
181 if ancestors_temp:
182 ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
183 working_path = base_path+"/"+source_path
184 for idx in range(len(ancestors_temp)):
185 d = ancestors_temp[idx]
186 working_path = working_path.replace(d['path'], d['copyfrom_path'])
187 working_rev = d['copyfrom_rev']
188 ancestors.append({'path': working_path, 'revision': working_rev})
189 max_len = 0
190 for idx in range(len(ancestors)):
191 d = ancestors[idx]
192 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
193 ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
194 for idx in range(len(ancestors)-1):
195 d = ancestors[idx]
196 d_next = ancestors[idx+1]
197 ui.status(prefix + " [%s] %s <-- %s", idx,
198 str(d['path']+"@"+str(d['revision'])).ljust(max_len),
199 str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
200 level=ui.DEBUG, color='YELLOW')
201 else:
202 ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
203 svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
204 return ancestors
205
206 def get_rev_map(rev_map, src_rev, prefix):
207 """
208 Find the equivalent rev # in the target repo for the given rev # from the source repo.
209 """
210 ui.status(prefix + ">> get_rev_map(%s)", src_rev, level=ui.DEBUG, color='GREEN')
211 # Find the highest entry less-than-or-equal-to src_rev
212 for rev in range(src_rev, 0, -1):
213 ui.status(prefix + ">> get_rev_map: rev=%s in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
214 if rev in rev_map:
215 return rev_map[rev]
216 # Else, we fell off the bottom of the rev_map. Ruh-roh...
217 return None
218
219 def get_svn_dirlist(svn_path, svn_rev = ""):
220 """
221 Get a list of all the child contents (recusive) of the given folder path.
222 """
223 args = ["list"]
224 path = svn_path
225 if svn_rev:
226 args += ["-r", svn_rev]
227 path += "@"+str(svn_rev)
228 args += [path]
229 paths = run_svn(args, no_fail=True)
230 paths = paths.strip("\n").split("\n") if len(paths)>1 else []
231 return paths
232
233 def _add_export_path(export_paths, path_offset):
234 found = False
235 for p in export_paths:
236 if path_offset.startswith(p):
237 found = True
238 break
239 if not found:
240 export_paths.append(path_offset)
241 return export_paths
242
243 def do_svn_add(source_repos_url, source_url, path_offset, target_url, source_rev, \
244 parent_copyfrom_path="", parent_copyfrom_rev="", export_paths={}, \
245 rev_map={}, is_dir = False, prefix = ""):
246 """
247 Given the add'd source path, replay the "svn add/copy" commands to correctly
248 track renames across copy-from's.
249
250 For example, consider a sequence of events like this:
251 1. svn copy /trunk /branches/fix1
252 2. (Make some changes on /branches/fix1)
253 3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2 " Rename folder
254 4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt " Rename file inside renamed folder
255 5. svn co /trunk && svn merge /branches/fix1
256 After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
257 and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
258 to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
259 we'd lose the logical history that Proj2/file2.txt is really a descendant
260 of Proj1/file1.txt.
261
262 'source_repos_url' is the full URL to the root of the source repository.
263 'source_url' is the full URL to the source path in the source repository.
264 'path_offset' is the offset from source_base to the file to check ancestry for,
265 e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
266 'target_url' is the full URL to the target path in the target repository.
267 'source_rev' is the revision ("svn log") that we're processing from the source repo.
268 'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
269 directory, when being called recursively by do_svn_add_dir().
270 'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
271 'rev_map' is the running mapping-table dictionary for source-repo rev #'s
272 to the equivalent target-repo rev #'s.
273 'is_dir' is whether path_offset is a directory (rather than a file).
274 """
275 source_base = source_url[len(source_repos_url):]
276 ui.status(prefix + ">> do_svn_add: %s %s", source_base+"/"+path_offset+"@"+str(source_rev),
277 " (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
278 level=ui.DEBUG, color='GREEN')
279 # Check if the given path has ancestors which chain back to the current source_base
280 found_ancestor = False
281 ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+" ")
282 # ancestors[n] is the original (pre-branch-copy) trunk path.
283 # ancestors[n-1] is the first commit on the new branch.
284 copyfrom_path = ancestors[len(ancestors)-1]['path'] if ancestors else ""
285 copyfrom_rev = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
286 if ancestors:
287 # The copy-from path has ancestory back to source_url.
288 ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
289 level=ui.DEBUG, color='GREEN', bold=True)
290 found_ancestor = True
291 # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
292 # return None in the case where copyfrom_rev is *before* our source_start_rev.
293 tgt_rev = get_rev_map(rev_map, copyfrom_rev, prefix+" ")
294 ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
295 else:
296 ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
297 found_ancestor = False
298 if found_ancestor and tgt_rev:
299 # Check if this path_offset in the target WC already has this ancestry, in which
300 # case there's no need to run the "svn copy" (again).
301 path_in_svn = in_svn(path_offset, prefix=prefix+" ")
302 log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+" ") else []
303 if (not log_entry or (log_entry['revision'] != tgt_rev)):
304 copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
305 ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
306 ui.status(prefix + " copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
307 ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
308 if path_in_svn and \
309 ((parent_copyfrom_path and copyfrom_path.startswith(parent_copyfrom_path)) and \
310 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
311 # When being called recursively, if this child entry has the same ancestor as the
312 # the parent, then no need to try to run another "svn copy".
313 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
314 parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
315 pass
316 else:
317 # Copy this path from the equivalent path+rev in the target repo, to create the
318 # equivalent history.
319 if parent_copyfrom_path:
320 # If we have a parent copy-from path, we mis-match that so display a status
321 # message describing the action we're mimic'ing. If path_in_svn, then this
322 # is logically a "replace" rather than an "add".
323 ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
324 if path_in_svn:
325 # If local file is already under version-control, then this is a replace.
326 ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
327 run_svn(["remove", "--force", path_offset])
328 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
329 # Export the final version of this file/folder from the source repo, to make
330 # sure we're up-to-date.
331 export_paths = _add_export_path(export_paths, path_offset)
332 else:
333 ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
334 else:
335 # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
336 # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
337 # Create (parent) directory if needed.
338 # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
339 # split-out to a shared tag?
340 p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
341 if not os.path.exists(p_path):
342 run_svn(["mkdir", p_path])
343 if not in_svn(path_offset, prefix=prefix+" "):
344 if is_dir:
345 # Export the final verison of all files in this folder.
346 export_paths = _add_export_path(export_paths, path_offset)
347 else:
348 # Export the final verison of this file. We *need* to do this before running
349 # the "svn add", even if we end-up re-exporting this file again via export_paths.
350 run_svn(["export", "--force", "-r", source_rev,
351 source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
352 # If not already under version-control, then "svn add" this file/folder.
353 run_svn(["add", "--parents", path_offset])
354 # TODO: Need to copy SVN properties from source repos
355 if is_dir:
356 # For any folders that we process, process any child contents, so that we correctly
357 # replay copies/replaces/etc.
358 do_svn_add_dir(source_repos_url, source_url, path_offset, source_rev, target_url,
359 copyfrom_path, copyfrom_rev, export_paths, rev_map, prefix+" ")
360
361 def do_svn_add_dir(source_repos_url, source_url, path_offset, source_rev, target_url, \
362 parent_copyfrom_path, parent_copyfrom_rev, export_paths, rev_map, prefix=""):
363 source_base = source_url[len(source_repos_url):]
364 # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
365 # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
366 # associated remote repo folder. (Is this a problem?)
367 paths_local = get_svn_dirlist(path_offset)
368 paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
369 ui.status(prefix + ">> do_svn_add_dir: paths_local: %s", str(paths_local), level=ui.DEBUG, color='GREEN')
370 ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
371 # Update files/folders which exist in remote but not local
372 for path in paths_remote:
373 path_is_dir = True if path[-1] == "/" else False
374 working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
375 do_svn_add(source_repos_url, source_url, working_path, target_url, source_rev,
376 parent_copyfrom_path, parent_copyfrom_rev, export_paths,
377 rev_map, path_is_dir, prefix+" ")
378 # Remove files/folders which exist in local but not remote
379 for path in paths_local:
380 if not path in paths_remote:
381 ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
382 run_svn(["remove", "--force", path_offset+"/"+path])
383 # TODO: Does this handle deleted folders too? Wouldn't want to have a case
384 # where we only delete all files from folder but leave orphaned folder around.
385
386 def process_svn_log_entry(log_entry, source_repos_url, source_url, target_url, \
387 rev_map, commit_paths = [], prefix = ""):
388 """
389 Process SVN changes from the given log entry.
390 Returns array of all the paths in the working-copy that were changed,
391 i.e. the paths which need to be "svn commit".
392
393 'log_entry' is the array structure built by parse_svn_log_xml().
394 'source_repos_url' is the full URL to the root of the source repository.
395 'source_url' is the full URL to the source path in the source repository.
396 'target_url' is the full URL to the target path in the target repository.
397 'rev_map' is the running mapping-table dictionary for source-repo rev #'s
398 to the equivalent target-repo rev #'s.
399 'commit_paths' is the working list of specific paths which changes to pass
400 to the final "svn commit".
401 """
402 removed_paths = []
403 export_paths = []
404 # Get the relative offset of source_url based on source_repos_url
405 # e.g. '/branches/bug123'
406 source_base = source_url[len(source_repos_url):]
407 source_rev = log_entry['revision']
408 ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
409 for d in log_entry['changed_paths']:
410 # Get the full path for this changed_path
411 # e.g. '/branches/bug123/projectA/file1.txt'
412 path = d['path']
413 if not path.startswith(source_base + "/"):
414 # Ignore changed files that are not part of this subdir
415 if path != source_base:
416 ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
417 continue
418 # Calculate the offset (based on source_base) for this changed_path
419 # e.g. 'projectA/file1.txt'
420 # (path = source_base + "/" + path_offset)
421 path_offset = path[len(source_base):].strip("/")
422 # Get the action for this path
423 action = d['action']
424 if action not in 'MARD':
425 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
426 % (source_rev, action))
427 if action not in 'D':
428 # (Note: Skip displaying action message for 'D' here since we'll display that
429 # message when we process the deferred delete actions at the end.)
430 ui.status(" %s %s%s", action, d['path'],
431 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
432 level=ui.VERBOSE)
433
434 # Try to be efficient and keep track of an explicit list of paths in the
435 # working copy that changed. If we commit from the root of the working copy,
436 # then SVN needs to crawl the entire working copy looking for pending changes.
437 # But, if we gather too many paths to commit, then we wipe commit_paths below
438 # and end-up doing a commit at the root of the working-copy.
439 if len (commit_paths) < 100:
440 commit_paths.append(path_offset)
441
442 # Special-handling for replace's
443 if action == 'R':
444 # If file was "replaced" (deleted then re-added, all in same revision),
445 # then we need to run the "svn rm" first, then change action='A'. This
446 # lets the normal code below handle re-"svn add"'ing the files. This
447 # should replicate the "replace".
448 run_svn(["remove", "--force", path_offset])
449 action = 'A'
450
451 # Handle all the various action-types
452 # (Handle "add" first, for "svn copy/move" support)
453 if action == 'A':
454 # If we have any queued deletions for this same path, remove those if we're re-adding this path.
455 if path_offset in removed_paths:
456 removed_paths.remove(path_offset)
457 # Determine where to export from.
458 svn_copy = False
459 path_is_dir = True if d['kind'] == 'dir' else False
460 # Handle cases where this "add" was a copy from another URL in the source repos
461 if d['copyfrom_revision']:
462 copyfrom_path = d['copyfrom_path']
463 copyfrom_rev = d['copyfrom_revision']
464 do_svn_add(source_repos_url, source_url, path_offset, target_url, source_rev,
465 "", "", export_paths, rev_map, path_is_dir, prefix+" ")
466 # Else just "svn export" the files from the source repo and "svn add" them.
467 else:
468 # Create (parent) directory if needed
469 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
470 if not os.path.exists(p_path):
471 run_svn(["mkdir", p_path])
472 # Export the entire added tree.
473 if path_is_dir:
474 export_paths = _add_export_path(export_paths, path_offset)
475 else:
476 # Export the final verison of this file. We *need* to do this before running
477 # the "svn add", even if we end-up re-exporting this file again via export_paths.
478 run_svn(["export", "--force", "-r", source_rev,
479 source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
480 if not in_svn(path_offset, prefix=prefix+" "):
481 # Need to use in_svn here to handle cases where client committed the parent
482 # folder and each indiv sub-folder.
483 run_svn(["add", "--parents", path_offset])
484 # TODO: Need to copy SVN properties from source repos
485
486 elif action == 'D':
487 # Queue "svn remove" commands, to allow the action == 'A' handling the opportunity
488 # to do smart "svn copy" handling on copy/move/renames.
489 if not path_offset in removed_paths:
490 removed_paths.append(path_offset)
491
492 elif action == 'M':
493 # TODO: Is "svn merge -c" correct here? Should this just be an "svn export" plus
494 # proplist updating?
495 out = run_svn(["merge", "-c", source_rev, "--non-recursive",
496 "--non-interactive", "--accept=theirs-full",
497 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
498
499 else:
500 raise SVNError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
501 % action)
502
503 # Process any deferred removed actions
504 if removed_paths:
505 path_base = source_url[len(source_repos_url):]
506 for path_offset in removed_paths:
507 ui.status(" %s %s", 'D', path_base+"/"+path_offset, level=ui.VERBOSE)
508 run_svn(["remove", "--force", path_offset])
509 # Export the final version of all add'd paths from source_url
510 if export_paths:
511 for path_offset in export_paths:
512 run_svn(["export", "--force", "-r", source_rev,
513 source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
514
515 return commit_paths
516
517 def disp_svn_log_summary(log_entry):
518 ui.status("")
519 ui.status("r%s | %s | %s",
520 log_entry['revision'],
521 log_entry['author'],
522 str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
523 ui.status(log_entry['message'])
524 ui.status("------------------------------------------------------------------------")
525
526 def pull_svn_rev(log_entry, source_repos_url, source_repos_uuid, source_url, target_url, rev_map, keep_author=False):
527 """
528 Pull SVN changes from the given log entry.
529 Returns the new SVN revision.
530 If an exception occurs, it will rollback to revision 'source_rev - 1'.
531 """
532 disp_svn_log_summary(log_entry)
533 source_rev = log_entry['revision']
534
535 # Process all the paths in this log entry
536 commit_paths = []
537 process_svn_log_entry(log_entry, source_repos_url, source_url, target_url,
538 rev_map, commit_paths)
539 # If we had too many individual paths to commit, wipe the list and just commit at
540 # the root of the working copy.
541 if len (commit_paths) > 99:
542 commit_paths = []
543
544 # Add source-tracking revprop's
545 revprops = [{'name':'source_uuid', 'value':source_repos_uuid},
546 {'name':'source_url', 'value':source_url},
547 {'name':'source_rev', 'value':source_rev}]
548 commit_from_svn_log_entry(log_entry, commit_paths, keep_author=keep_author, revprops=revprops)
549
550 def run_parser(parser):
551 """
552 Add common options to an OptionParser instance, and run parsing.
553 """
554 parser.add_option("", "--version", dest="show_version", action="store_true",
555 help="show version and exit")
556 parser.remove_option("--help")
557 parser.add_option("-h", "--help", dest="show_help", action="store_true",
558 help="show this help message and exit")
559 parser.add_option("-v", "--verbose", dest="verbosity", const=ui.VERBOSE,
560 default=10, action="store_const",
561 help="enable additional output")
562 parser.add_option("--debug", dest="verbosity", const=ui.DEBUG,
563 action="store_const",
564 help="enable debugging output")
565 options, args = parser.parse_args()
566 if options.show_help:
567 parser.print_help()
568 sys.exit(0)
569 if options.show_version:
570 prog_name = os.path.basename(sys.argv[0])
571 print prog_name, full_version
572 sys.exit(0)
573 ui.update_config(options)
574 return options, args
575
576 def display_parser_error(parser, message):
577 """
578 Display an options error, and terminate.
579 """
580 print "error:", message
581 print
582 parser.print_help()
583 sys.exit(1)
584
585 def real_main(options, args):
586 source_url = args.pop(0).rstrip("/")
587 target_url = args.pop(0).rstrip("/")
588 if options.keep_author:
589 keep_author = True
590 else:
591 keep_author = False
592
593 # Find the greatest_rev in the source repo
594 svn_info = svnclient.get_svn_info(source_url)
595 greatest_rev = svn_info['revision']
596 # Get the base URL for the source repos, e.g. 'svn://svn.example.com/svn/repo'
597 source_repos_url = svn_info['repos_url']
598 # Get the UUID for the source repos
599 source_repos_uuid = svn_info['repos_uuid']
600
601 wc_target = "_wc_target"
602 rev_map = {}
603
604 # if old working copy does not exist, disable continue mode
605 # TODO: Better continue support. Maybe include source repo's rev # in target commit info?
606 if not os.path.exists(wc_target):
607 options.cont_from_break = False
608
609 if not options.cont_from_break:
610 # Get log entry for the SVN revision we will check out
611 if options.svn_rev:
612 # If specify a rev, get log entry just before or at rev
613 svn_start_log = svnclient.get_last_svn_log_entry(source_url, 1, options.svn_rev, False)
614 else:
615 # Otherwise, get log entry of branch creation
616 # TODO: This call is *very* expensive on a repo with lots of revisions.
617 # Even though the call is passing --limit 1, it seems like that limit-filter
618 # is happening after SVN has fetched the full log history.
619 svn_start_log = svnclient.get_first_svn_log_entry(source_url, 1, greatest_rev, False)
620
621 # This is the revision we will start from for source_url
622 source_start_rev = svn_start_log['revision']
623
624 # Check out a working copy of target_url
625 wc_target = os.path.abspath(wc_target)
626 if os.path.exists(wc_target):
627 shutil.rmtree(wc_target)
628 svnclient.svn_checkout(target_url, wc_target)
629 os.chdir(wc_target)
630
631 # For the initial commit to the target URL, export all the contents from
632 # the source URL at the start-revision.
633 paths = run_svn(["list", "-r", source_start_rev, source_url+"@"+str(source_start_rev)])
634 if len(paths)>1:
635 disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_start_rev, source_start_rev))
636 ui.status("(Initial import)", level=ui.VERBOSE)
637 paths = paths.strip("\n").split("\n")
638 for path in paths:
639 # For each top-level file/folder...
640 if not path:
641 # Skip null lines
642 break
643 # Directories have a trailing slash in the "svn list" output
644 path_is_dir = True if path[-1] == "/" else False
645 if path_is_dir:
646 path=path.rstrip('/')
647 if not os.path.exists(path):
648 os.makedirs(path)
649 run_svn(["export", "--force", "-r" , source_start_rev, source_url+"/"+path+"@"+str(source_start_rev), path])
650 run_svn(["add", path])
651 revprops = [{'name':'source_uuid', 'value':source_repos_uuid},
652 {'name':'source_url', 'value':source_url},
653 {'name':'source_rev', 'value':source_start_rev}]
654 commit_from_svn_log_entry(svn_start_log, [], keep_author=keep_author, revprops=revprops)
655 else:
656 wc_target = os.path.abspath(wc_target)
657 os.chdir(wc_target)
658 # TODO: Need better resume support. For the time being, expect caller explictly passes in resume revision.
659 source_start_rev = options.svn_rev
660 if source_start_rev < 1:
661 display_error("Invalid arguments\n\nNeed to pass result rev # (-r) when using continue-mode (-c)", False)
662
663 # Load SVN log starting from source_start_rev + 1
664 it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev + 1, greatest_rev)
665
666 try:
667 for log_entry in it_log_entries:
668 # Replay this revision from source_url into target_url
669 pull_svn_rev(log_entry, source_repos_url, source_repos_uuid, source_url,
670 target_url, rev_map, keep_author)
671 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
672 run_svn(["update"])
673 # TODO: Run "svn cleanup" every 50 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
674 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
675 dup_info = svnclient.get_svn_info(target_url)
676 dup_rev = dup_info['revision']
677 source_rev = log_entry['revision']
678 ui.status(">> main: rev_map.add: source_rev=%s target_rev=%s", source_rev, dup_rev, level=ui.DEBUG, color='GREEN')
679 rev_map[source_rev] = dup_rev
680
681 except KeyboardInterrupt:
682 print "\nStopped by user."
683 run_svn(["cleanup"])
684 run_svn(["revert", "--recursive", "."])
685 # TODO: Run "svn status" and pro-actively delete any "?" orphaned entries, to clean-up the WC?
686 except:
687 print "\nCommand failed with following error:\n"
688 traceback.print_exc()
689 run_svn(["cleanup"])
690 print run_svn(["status"])
691 run_svn(["revert", "--recursive", "."])
692 # TODO: Run "svn status" and pro-actively delete any "?" orphaned entries, to clean-up the WC?
693 finally:
694 run_svn(["update"])
695 print "\nFinished!"
696
697 def main():
698 # Defined as entry point. Must be callable without arguments.
699 usage = "Usage: %prog [OPTIONS] source_url target_url"
700 parser = OptionParser(usage)
701 parser.add_option("-r", "--revision", type="int", dest="svn_rev", metavar="REV",
702 help="initial SVN revision to checkout from")
703 parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author",
704 help="maintain original Author info from source repo")
705 parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
706 help="continue from previous break")
707 (options, args) = run_parser(parser)
708 if len(args) != 2:
709 display_parser_error(parser, "incorrect number of arguments")
710 return real_main(options, args)
711
712
713 if __name__ == "__main__":
714 sys.exit(main() or 0)