]> Tony Duckles's Git Repositories (git.nynim.org) - svn2svn.git/blob - svn2svn/run/svn2svn.py
Fix copy-from handling and parent-path checking in get_kind()
[svn2svn.git] / svn2svn / run / svn2svn.py
1 """
2 Replicate (replay) changesets from one SVN repository to another.
3 """
4
5 from .. import base_version, full_version
6 from .. import ui
7 from .. import svnclient
8 from ..shell import run_svn
9 from ..errors import (ExternalCommandFailed, UnsupportedSVNAction, InternalError, VerificationError)
10 from parse import HelpFormatter
11
12 import sys
13 import os
14 import time
15 import traceback
16 import shutil
17 import operator
18 import optparse
19 from datetime import datetime
20
21 _valid_svn_actions = "MARD" # The list of known SVN action abbr's, from "svn log"
22
23 # Module-level variables/parameters
24 source_url = "" # URL to source path in source SVN repo, e.g. 'http://server/svn/source/trunk'
25 source_repos_url = "" # URL to root of source SVN repo, e.g. 'http://server/svn/source'
26 source_base = "" # Relative path of source_url in source SVN repo, e.g. '/trunk'
27 source_repos_uuid = "" # UUID of source SVN repo
28 target_url ="" # URL to target path in target SVN repo, e.g. 'file:///svn/repo_target/trunk'
29 rev_map = {} # The running mapping-table dictionary for source_url rev #'s -> target_url rev #'s
30
31 def commit_from_svn_log_entry(log_entry, options, commit_paths=None, target_revprops=None):
32 """
33 Given an SVN log entry and an optional list of changed paths, do an svn commit.
34 """
35 # TODO: Run optional external shell hook here, for doing pre-commit filtering
36 # Display the _wc_target "svn status" info if running in -vv (or higher) mode
37 if ui.get_level() >= ui.EXTRA:
38 ui.status(">> commit_from_svn_log_entry: Pre-commit _wc_target status:", level=ui.EXTRA, color='CYAN')
39 ui.status(run_svn(["status"]), level=ui.EXTRA, color='CYAN')
40 # This will use the local timezone for displaying commit times
41 timestamp = int(log_entry['date'])
42 svn_date = str(datetime.fromtimestamp(timestamp))
43 # Uncomment this one one if you prefer UTC commit times
44 #svn_date = "%d 0" % timestamp
45 args = ["commit", "--force-log"]
46 if options.keep_author:
47 args += ["-m", log_entry['message'] + "\nDate: " + svn_date, "--username", log_entry['author']]
48 else:
49 args += ["-m", log_entry['message'] + "\nDate: " + svn_date + "\nAuthor: " + log_entry['author']]
50 revprops = {}
51 if log_entry['revprops']:
52 # Carry forward any revprop's from the source revision
53 for v in log_entry['revprops']:
54 revprops[v['name']] = v['value']
55 if target_revprops:
56 # Add any extra revprop's we want to set for the target repo commits
57 for v in target_revprops:
58 revprops[v['name']] = v['value']
59 if revprops:
60 for key in revprops:
61 args += ["--with-revprop", "%s=%s" % (key, str(revprops[key]))]
62 if commit_paths:
63 if len(commit_paths)<100:
64 # If we don't have an excessive amount of individual changed paths, pass
65 # those to the "svn commit" command. Else, pass nothing so we commit at
66 # the root of the working-copy.
67 args += list(commit_paths)
68 rev = None
69 if not options.dry_run:
70 # Run the "svn commit" command, and screen-scrape the target_rev value (if any)
71 output = run_svn(args)
72 if output:
73 output_lines = output.strip("\n").split("\n")
74 rev = ""
75 for line in output_lines:
76 if line[0:19] == 'Committed revision ':
77 rev = line[19:].rstrip('.')
78 break
79 if rev:
80 ui.status("Committed revision %s.", rev)
81 return rev
82
83 def full_svn_revert():
84 """
85 Do an "svn revert" and proactively remove any extra files in the working copy.
86 """
87 run_svn(["revert", "--recursive", "."])
88 output = run_svn(["status"])
89 if output:
90 output_lines = output.strip("\n").split("\n")
91 for line in output_lines:
92 if line[0] == "?":
93 path = line[4:].strip(" ")
94 if os.path.isfile(path):
95 os.remove(path)
96 if os.path.isdir(path):
97 shutil.rmtree(path)
98
99 def gen_tracking_revprops(source_rev):
100 """
101 Build an array of svn2svn-specific source-tracking revprops.
102 """
103 revprops = [{'name':'svn2svn:source_uuid', 'value':source_repos_uuid},
104 {'name':'svn2svn:source_url', 'value':source_url},
105 {'name':'svn2svn:source_rev', 'value':source_rev}]
106 return revprops
107
108 def in_svn(p, require_in_repo=False, prefix=""):
109 """
110 Check if a given file/folder is being tracked by Subversion.
111 Prior to SVN 1.6, we could "cheat" and look for the existence of ".svn" directories.
112 With SVN 1.7 and beyond, WC-NG means only a single top-level ".svn" at the root of the working-copy.
113 Use "svn status" to check the status of the file/folder.
114 """
115 entries = svnclient.get_svn_status(p, no_recursive=True)
116 if not entries:
117 return False
118 d = entries[0]
119 if require_in_repo and (d['status'] == 'added' or d['revision'] is None):
120 # If caller requires this path to be in the SVN repo, prevent returning True
121 # for paths that are only locally-added.
122 ret = False
123 else:
124 # Don't consider files tracked as deleted in the WC as under source-control.
125 # Consider files which are locally added/copied as under source-control.
126 ret = True if not (d['status'] == 'deleted') and (d['type'] == 'normal' or d['status'] == 'added' or d['copied'] == 'true') else False
127 ui.status(prefix + ">> in_svn('%s', require_in_repo=%s) --> %s", p, str(require_in_repo), str(ret), level=ui.DEBUG, color='GREEN')
128 return ret
129
130 def is_child_path(path, p_path):
131 return True if (path == p_path) or (path.startswith(p_path+"/")) else False
132
133 def find_svn_ancestors(svn_repos_url, base_path, source_path, source_rev, prefix = ""):
134 """
135 Given a source path, walk the SVN history backwards to inspect the ancestory of
136 that path, seeing if it traces back to base_path. Build an array of copyfrom_path
137 and copyfrom_revision pairs for each of the "svn copies". If we find a copyfrom_path
138 which base_path is a substring match of (e.g. we crawled back to the initial branch-
139 copy from trunk), then return the collection of ancestor paths. Otherwise,
140 copyfrom_path has no ancestory compared to base_path.
141
142 This is useful when comparing "trunk" vs. "branch" paths, to handle cases where a
143 file/folder was renamed in a branch and then that branch was merged back to trunk.
144
145 'svn_repos_url' is the full URL to the root of the SVN repository,
146 e.g. 'file:///path/to/repo'
147 'base_path' is the path in the SVN repo to the target path we're trying to
148 trace ancestry back to, e.g. 'trunk'.
149 'source_path' is the path in the SVN repo to the source path to start checking
150 ancestry at, e.g. 'branches/fix1/projectA/file1.txt'.
151 (full_path = svn_repos_url+base_path+"/"+path_offset)
152 'source_rev' is the revision to start walking the history of source_path backwards from.
153 """
154 ui.status(prefix + ">> find_svn_ancestors: Start: (%s) source_path: %s base_path: %s",
155 svn_repos_url, source_path+"@"+str(source_rev), base_path, level=ui.DEBUG, color='YELLOW')
156 done = False
157 working_path = base_path+"/"+source_path
158 working_rev = source_rev
159 first_iter_done = False
160 ancestors_temp = []
161 while not done:
162 # Get the first "svn log" entry for this path (relative to @rev)
163 ui.status(prefix + ">> find_svn_ancestors: %s", svn_repos_url + working_path+"@"+str(working_rev), level=ui.DEBUG, color='YELLOW')
164 log_entry = svnclient.get_first_svn_log_entry(svn_repos_url + working_path, 1, working_rev, True)
165 if not log_entry:
166 ui.status(prefix + ">> find_svn_ancestors: Done: no log_entry", level=ui.DEBUG, color='YELLOW')
167 done = True
168 break
169 # If we found a copy-from case which matches our base_path, we're done.
170 # ...but only if we've at least tried to search for the first copy-from path.
171 if first_iter_done and is_child_path(working_path, base_path):
172 ui.status(prefix + ">> find_svn_ancestors: Done: Found is_child_path(working_path, base_path) and first_iter_done=True", level=ui.DEBUG, color='YELLOW')
173 done = True
174 break
175 first_iter_done = True
176 # Search for any actions on our target path (or parent paths).
177 changed_paths_temp = []
178 for d in log_entry['changed_paths']:
179 path = d['path']
180 if path in working_path:
181 changed_paths_temp.append({'path': path, 'data': d})
182 if not changed_paths_temp:
183 # If no matches, then we've hit the end of the chain and this path has no ancestry back to base_path.
184 ui.status(prefix + ">> find_svn_ancestors: Done: No matching changed_paths", level=ui.DEBUG, color='YELLOW')
185 done = True
186 continue
187 # Reverse-sort any matches, so that we start with the most-granular (deepest in the tree) path.
188 changed_paths = sorted(changed_paths_temp, key=operator.itemgetter('path'), reverse=True)
189 # Find the action for our working_path in this revision. Use a loop to check in reverse order,
190 # so that if the target file/folder is "M" but has a parent folder with an "A" copy-from.
191 for v in changed_paths:
192 d = v['data']
193 path = d['path']
194 # Check action-type for this file
195 action = d['action']
196 if action not in _valid_svn_actions:
197 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
198 % (log_entry['revision'], action))
199 ui.status(prefix + "> %s %s%s", action, path,
200 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
201 level=ui.DEBUG, color='YELLOW')
202 if action == 'D':
203 # If file/folder was deleted, it has no ancestor
204 ancestors_temp = []
205 ui.status(prefix + ">> find_svn_ancestors: Done: deleted", level=ui.DEBUG, color='YELLOW')
206 done = True
207 break
208 if action in 'RA':
209 # If file/folder was added/replaced but not a copy, it has no ancestor
210 if not d['copyfrom_path']:
211 ancestors_temp = []
212 ui.status(prefix + ">> find_svn_ancestors: Done: %s with no copyfrom_path",
213 "Added" if action == "A" else "Replaced",
214 level=ui.DEBUG, color='YELLOW')
215 done = True
216 break
217 # Else, file/folder was added/replaced and is a copy, so add an entry to our ancestors list
218 # and keep checking for ancestors
219 ui.status(prefix + ">> find_svn_ancestors: Found copy-from (action=%s): %s --> %s",
220 action, path, d['copyfrom_path']+"@"+str(d['copyfrom_revision']),
221 level=ui.DEBUG, color='YELLOW')
222 ancestors_temp.append({'path': path, 'revision': log_entry['revision'],
223 'copyfrom_path': d['copyfrom_path'], 'copyfrom_rev': d['copyfrom_revision']})
224 working_path = working_path.replace(d['path'], d['copyfrom_path'])
225 working_rev = d['copyfrom_revision']
226 # Follow the copy and keep on searching
227 break
228 ancestors = []
229 if ancestors_temp:
230 ancestors.append({'path': base_path+"/"+source_path, 'revision': source_rev})
231 working_path = base_path+"/"+source_path
232 for idx in range(len(ancestors_temp)):
233 d = ancestors_temp[idx]
234 working_path = working_path.replace(d['path'], d['copyfrom_path'])
235 working_rev = d['copyfrom_rev']
236 ancestors.append({'path': working_path, 'revision': working_rev})
237 if ui.get_level() >= ui.DEBUG:
238 max_len = 0
239 for idx in range(len(ancestors)):
240 d = ancestors[idx]
241 max_len = max(max_len, len(d['path']+"@"+str(d['revision'])))
242 ui.status(prefix + ">> find_svn_ancestors: Found parent ancestors:", level=ui.DEBUG, color='YELLOW_B')
243 for idx in range(len(ancestors)-1):
244 d = ancestors[idx]
245 d_next = ancestors[idx+1]
246 ui.status(prefix + " [%s] %s <-- %s", idx,
247 str(d['path']+"@"+str(d['revision'])).ljust(max_len),
248 str(d_next['path']+"@"+str(d_next['revision'])).ljust(max_len),
249 level=ui.DEBUG, color='YELLOW')
250 else:
251 ui.status(prefix + ">> find_svn_ancestors: No ancestor-chain found: %s",
252 svn_repos_url+base_path+"/"+source_path+"@"+str(source_rev), level=ui.DEBUG, color='YELLOW')
253 return ancestors
254
255 def get_rev_map(source_rev, prefix):
256 """
257 Find the equivalent rev # in the target repo for the given rev # from the source repo.
258 """
259 ui.status(prefix + ">> get_rev_map(%s)", source_rev, level=ui.DEBUG, color='GREEN')
260 # Find the highest entry less-than-or-equal-to source_rev
261 for rev in range(int(source_rev), 0, -1):
262 ui.status(prefix + ">> get_rev_map: rev=%s in_rev_map=%s", rev, str(rev in rev_map), level=ui.DEBUG, color='BLACK_B')
263 if rev in rev_map:
264 return int(rev_map[rev])
265 # Else, we fell off the bottom of the rev_map. Ruh-roh...
266 return None
267
268 def set_rev_map(source_rev, target_rev):
269 ui.status(">> set_rev_map: source_rev=%s target_rev=%s", source_rev, target_rev, level=ui.DEBUG, color='GREEN')
270 global rev_map
271 rev_map[int(source_rev)]=int(target_rev)
272
273 def build_rev_map(target_url, target_end_rev, source_info):
274 """
275 Check for any already-replayed history from source_url (source_info) and
276 build the mapping-table of source_rev -> target_rev.
277 """
278 global rev_map
279 rev_map = {}
280 ui.status("Rebuilding rev_map...", level=ui.VERBOSE)
281 proc_count = 0
282 it_log_entries = svnclient.iter_svn_log_entries(target_url, 1, target_end_rev, get_changed_paths=False, get_revprops=True)
283 for log_entry in it_log_entries:
284 if log_entry['revprops']:
285 revprops = {}
286 for v in log_entry['revprops']:
287 if v['name'].startswith('svn2svn:'):
288 revprops[v['name']] = v['value']
289 if revprops and \
290 revprops['svn2svn:source_uuid'] == source_info['repos_uuid'] and \
291 revprops['svn2svn:source_url'] == source_info['url']:
292 source_rev = revprops['svn2svn:source_rev']
293 target_rev = log_entry['revision']
294 set_rev_map(source_rev, target_rev)
295
296 def get_svn_dirlist(svn_path, svn_rev = ""):
297 """
298 Get a list of all the child contents (recusive) of the given folder path.
299 """
300 args = ["list"]
301 path = svn_path
302 if svn_rev:
303 args += ["-r", svn_rev]
304 path += "@"+str(svn_rev)
305 args += [path]
306 paths = run_svn(args, no_fail=True)
307 paths = paths.strip("\n").split("\n") if len(paths)>1 else []
308 return paths
309
310 def path_in_list(paths, path):
311 for p in paths:
312 if is_child_path(path, p):
313 return True
314 return False
315
316 def add_path(paths, path):
317 if not path_in_list(paths, path):
318 paths.append(path)
319
320 def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom_rev="", \
321 export_paths={}, is_dir = False, prefix = ""):
322 """
323 Given the add'd source path, replay the "svn add/copy" commands to correctly
324 track renames across copy-from's.
325
326 For example, consider a sequence of events like this:
327 1. svn copy /trunk /branches/fix1
328 2. (Make some changes on /branches/fix1)
329 3. svn mv /branches/fix1/Proj1 /branches/fix1/Proj2 " Rename folder
330 4. svn mv /branches/fix1/Proj2/file1.txt /branches/fix1/Proj2/file2.txt " Rename file inside renamed folder
331 5. svn co /trunk && svn merge /branches/fix1
332 After the merge and commit, "svn log -v" with show a delete of /trunk/Proj1
333 and and add of /trunk/Proj2 copy-from /branches/fix1/Proj2. If we were just
334 to do a straight "svn export+add" based on the /branches/fix1/Proj2 folder,
335 we'd lose the logical history that Proj2/file2.txt is really a descendant
336 of Proj1/file1.txt.
337
338 'path_offset' is the offset from source_base to the file to check ancestry for,
339 e.g. 'projectA/file1.txt'. path = source_repos_url + source_base + path_offset.
340 'source_rev' is the revision ("svn log") that we're processing from the source repo.
341 'parent_copyfrom_path' and 'parent_copyfrom_rev' is the copy-from path of the parent
342 directory, when being called recursively by do_svn_add_dir().
343 'export_paths' is the list of path_offset's that we've deferred running "svn export" on.
344 'is_dir' is whether path_offset is a directory (rather than a file).
345 """
346 ui.status(prefix + ">> do_svn_add: %s %s", source_base+"/"+path_offset+"@"+str(source_rev),
347 " (parent-copyfrom: "+parent_copyfrom_path+"@"+str(parent_copyfrom_rev)+")" if parent_copyfrom_path else "",
348 level=ui.DEBUG, color='GREEN')
349 # Check if the given path has ancestors which chain back to the current source_base
350 found_ancestor = False
351 ancestors = find_svn_ancestors(source_repos_url, source_base, path_offset, source_rev, prefix+" ")
352 # ancestors[n] is the original (pre-branch-copy) trunk path.
353 # ancestors[n-1] is the first commit on the new branch.
354 copyfrom_path = ancestors[len(ancestors)-1]['path'] if ancestors else ""
355 copyfrom_rev = ancestors[len(ancestors)-1]['revision'] if ancestors else ""
356 if ancestors:
357 # The copy-from path has ancestory back to source_url.
358 ui.status(prefix + ">> do_svn_add: Check copy-from: Found parent: %s", copyfrom_path+"@"+str(copyfrom_rev),
359 level=ui.DEBUG, color='GREEN', bold=True)
360 found_ancestor = True
361 # Map the copyfrom_rev (source repo) to the equivalent target repo rev #. This can
362 # return None in the case where copyfrom_rev is *before* our source_start_rev.
363 tgt_rev = get_rev_map(copyfrom_rev, prefix+" ")
364 ui.status(prefix + ">> do_svn_add: get_rev_map: %s (source) -> %s (target)", copyfrom_rev, tgt_rev, level=ui.DEBUG, color='GREEN')
365 else:
366 ui.status(prefix + ">> do_svn_add: Check copy-from: No ancestor chain found.", level=ui.DEBUG, color='GREEN')
367 found_ancestor = False
368 if found_ancestor and tgt_rev:
369 # Check if this path_offset in the target WC already has this ancestry, in which
370 # case there's no need to run the "svn copy" (again).
371 path_in_svn = in_svn(path_offset, prefix=prefix+" ")
372 log_entry = svnclient.get_last_svn_log_entry(path_offset, 1, 'HEAD', get_changed_paths=False) if in_svn(path_offset, require_in_repo=True, prefix=prefix+" ") else []
373 if (not log_entry or (log_entry['revision'] != tgt_rev)):
374 copyfrom_offset = copyfrom_path[len(source_base):].strip('/')
375 ui.status(prefix + ">> do_svn_add: svn_copy: Copy-from: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
376 ui.status(prefix + " copyfrom: %s", copyfrom_path+"@"+str(copyfrom_rev), level=ui.DEBUG, color='GREEN')
377 ui.status(prefix + " p_copyfrom: %s", parent_copyfrom_path+"@"+str(parent_copyfrom_rev) if parent_copyfrom_path else "", level=ui.DEBUG, color='GREEN')
378 if path_in_svn and \
379 ((parent_copyfrom_path and is_child_path(copyfrom_path, parent_copyfrom_path)) and \
380 (parent_copyfrom_rev and copyfrom_rev == parent_copyfrom_rev)):
381 # When being called recursively, if this child entry has the same ancestor as the
382 # the parent, then no need to try to run another "svn copy".
383 ui.status(prefix + ">> do_svn_add: svn_copy: Same ancestry as parent: %s",
384 parent_copyfrom_path+"@"+str(parent_copyfrom_rev),level=ui.DEBUG, color='GREEN')
385 pass
386 else:
387 # Copy this path from the equivalent path+rev in the target repo, to create the
388 # equivalent history.
389 if parent_copyfrom_path:
390 # If we have a parent copy-from path, we mis-match that so display a status
391 # message describing the action we're mimic'ing. If path_in_svn, then this
392 # is logically a "replace" rather than an "add".
393 ui.status(" %s %s (from %s)", ('R' if path_in_svn else 'A'), source_base+"/"+path_offset, ancestors[1]['path']+"@"+str(copyfrom_rev), level=ui.VERBOSE)
394 if path_in_svn:
395 # If local file is already under version-control, then this is a replace.
396 ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN')
397 run_svn(["remove", "--force", path_offset])
398 run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset])
399 # Export the final version of this file/folder from the source repo, to make
400 # sure we're up-to-date.
401 add_path(export_paths, path_offset)
402 else:
403 ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN')
404 else:
405 # Else, either this copy-from path has no ancestry back to source_url OR copyfrom_rev comes
406 # before our initial source_start_rev (i.e. tgt_rev == None), so can't do a "svn copy".
407 # Create (parent) directory if needed.
408 # TODO: This is (nearly) a duplicate of code in process_svn_log_entry(). Should this be
409 # split-out to a shared tag?
410 p_path = path_offset if is_dir else os.path.dirname(path_offset).strip() or '.'
411 if not os.path.exists(p_path):
412 run_svn(["mkdir", p_path])
413 if not in_svn(path_offset, prefix=prefix+" "):
414 if is_dir:
415 # Export the final verison of all files in this folder.
416 add_path(export_paths, path_offset)
417 else:
418 # Export the final verison of this file. We *need* to do this before running
419 # the "svn add", even if we end-up re-exporting this file again via export_paths.
420 run_svn(["export", "--force", "-r", source_rev,
421 source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset])
422 # If not already under version-control, then "svn add" this file/folder.
423 run_svn(["add", "--parents", path_offset])
424 # TODO: Need to copy SVN properties from source repos
425 if is_dir:
426 # For any folders that we process, process any child contents, so that we correctly
427 # replay copies/replaces/etc.
428 do_svn_add_dir(path_offset, source_rev, copyfrom_path, copyfrom_rev, export_paths, prefix+" ")
429
430 def do_svn_add_dir(path_offset, source_rev, parent_copyfrom_path, parent_copyfrom_rev, \
431 export_paths, prefix=""):
432 # Get the directory contents, to compare between the local WC (target_url) vs. the remote repo (source_url)
433 # TODO: paths_local won't include add'd paths because "svn ls" lists the contents of the
434 # associated remote repo folder. (Is this a problem?)
435 paths_local = get_svn_dirlist(path_offset)
436 paths_remote = get_svn_dirlist(source_url+"/"+path_offset, source_rev)
437 ui.status(prefix + ">> do_svn_add_dir: paths_local: %s", str(paths_local), level=ui.DEBUG, color='GREEN')
438 ui.status(prefix + ">> do_svn_add_dir: paths_remote: %s", str(paths_remote), level=ui.DEBUG, color='GREEN')
439 # Update files/folders which exist in remote but not local
440 for path in paths_remote:
441 path_is_dir = True if path[-1] == "/" else False
442 working_path = path_offset+"/"+(path.rstrip('/') if path_is_dir else path)
443 do_svn_add(working_path, source_rev, parent_copyfrom_path, parent_copyfrom_rev,
444 export_paths, path_is_dir, prefix+" ")
445 # Remove files/folders which exist in local but not remote
446 for path in paths_local:
447 if not path in paths_remote:
448 ui.status(" %s %s", 'D', source_base+"/"+path_offset+"/"+path, level=ui.VERBOSE)
449 run_svn(["remove", "--force", path_offset+"/"+path])
450 # TODO: Does this handle deleted folders too? Wouldn't want to have a case
451 # where we only delete all files from folder but leave orphaned folder around.
452
453 def process_svn_log_entry(log_entry, options, commit_paths, prefix = ""):
454 """
455 Process SVN changes from the given log entry. Build an array (commit_paths)
456 of the paths in the working-copy that were changed, i.e. the paths which
457 we'll pass to "svn commit".
458 """
459 export_paths = []
460 source_rev = log_entry['revision']
461 ui.status(prefix + ">> process_svn_log_entry: %s", source_url+"@"+str(source_rev), level=ui.DEBUG, color='GREEN')
462 for d in log_entry['changed_paths']:
463 # Get the full path for this changed_path
464 # e.g. '/branches/bug123/projectA/file1.txt'
465 path = d['path']
466 if not is_child_path(path, source_base):
467 # Ignore changed files that are not part of this subdir
468 ui.status(prefix + ">> process_svn_log_entry: Unrelated path: %s (base: %s)", path, source_base, level=ui.DEBUG, color='GREEN')
469 continue
470 # Note: d['kind']="" for action="M" paths which only have property changes.
471 path_is_dir = True if d['kind'] == 'dir' else False
472 path_is_file = True if d['kind'] == 'file' else False
473 # Calculate the offset (based on source_base) for this changed_path
474 # e.g. 'projectA/file1.txt'
475 # (path = source_base + "/" + path_offset)
476 path_offset = path[len(source_base):].strip("/")
477 # Get the action for this path
478 action = d['action']
479 if action not in _valid_svn_actions:
480 raise UnsupportedSVNAction("In SVN rev. %d: action '%s' not supported. Please report a bug!"
481 % (source_rev, action))
482 ui.status(" %s %s%s", action, d['path'],
483 (" (from %s)" % (d['copyfrom_path']+"@"+str(d['copyfrom_revision']))) if d['copyfrom_path'] else "",
484 level=ui.VERBOSE)
485
486 # Try to be efficient and keep track of an explicit list of paths in the
487 # working copy that changed. If we commit from the root of the working copy,
488 # then SVN needs to crawl the entire working copy looking for pending changes.
489 commit_paths.append(path_offset)
490
491 # Special-handling for replace's
492 if action == 'R':
493 # If file was "replaced" (deleted then re-added, all in same revision),
494 # then we need to run the "svn rm" first, then change action='A'. This
495 # lets the normal code below handle re-"svn add"'ing the files. This
496 # should replicate the "replace".
497 run_svn(["remove", "--force", path_offset])
498 action = 'A'
499
500 # Handle all the various action-types
501 # (Handle "add" first, for "svn copy/move" support)
502 if action == 'A':
503 # Determine where to export from.
504 svn_copy = False
505 # Handle cases where this "add" was a copy from another URL in the source repo
506 if d['copyfrom_revision']:
507 copyfrom_path = d['copyfrom_path']
508 copyfrom_rev = d['copyfrom_revision']
509 do_svn_add(path_offset, source_rev, "", "", export_paths, path_is_dir, prefix+" ")
510 # Else just "svn export" the files from the source repo and "svn add" them.
511 else:
512 # Create (parent) directory if needed
513 p_path = path_offset if path_is_dir else os.path.dirname(path_offset).strip() or '.'
514 if not os.path.exists(p_path):
515 run_svn(["mkdir", p_path])
516 # Export the entire added tree.
517 if path_is_dir:
518 # For directories, defer the (recurisve) "svn export". Might have a
519 # situation in a branch merge where the entry in the svn-log is a
520 # non-copy-from'd "add" but there are child contents (that we haven't
521 # gotten to yet in log_entry) that are copy-from's. When we try do
522 # the "svn copy" later on in do_svn_add() for those copy-from'd paths,
523 # having pre-existing (svn-add'd) contents creates some trouble.
524 # Instead, just create the stub folders ("svn mkdir" above) and defer
525 # exporting the final file-state until the end.
526 add_path(export_paths, path_offset)
527 else:
528 # Export the final verison of this file. We *need* to do this before running
529 # the "svn add", even if we end-up re-exporting this file again via export_paths.
530 run_svn(["export", "--force", "-r", source_rev,
531 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
532 if not in_svn(path_offset, prefix=prefix+" "):
533 # Need to use in_svn here to handle cases where client committed the parent
534 # folder and each indiv sub-folder.
535 run_svn(["add", "--parents", path_offset])
536 # TODO: Need to copy SVN properties from source repos
537
538 elif action == 'D':
539 run_svn(["remove", "--force", path_offset])
540
541 elif action == 'M':
542 # TODO: Is "svn merge -c" correct here? Should this just be an "svn export" plus
543 # proplist updating?
544 out = run_svn(["merge", "-c", source_rev, "--non-recursive",
545 "--non-interactive", "--accept=theirs-full",
546 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
547 # TODO: If d['props'] == 'modified', then run code to clean-up/purge any newly-modified props?
548
549 else:
550 raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'"
551 % action)
552
553 # Export the final version of all add'd paths from source_url
554 if export_paths:
555 for path_offset in export_paths:
556 run_svn(["export", "--force", "-r", source_rev,
557 source_url+"/"+path_offset+"@"+str(source_rev), path_offset])
558
559 def disp_svn_log_summary(log_entry):
560 ui.status("")
561 ui.status("r%s | %s | %s",
562 log_entry['revision'],
563 log_entry['author'],
564 str(datetime.fromtimestamp(int(log_entry['date'])).isoformat(' ')))
565 ui.status(log_entry['message'])
566 ui.status("------------------------------------------------------------------------")
567
568 def real_main(options, args):
569 global source_url, target_url, rev_map
570 source_url = args.pop(0).rstrip("/") # e.g. 'http://server/svn/source/trunk'
571 target_url = args.pop(0).rstrip("/") # e.g. 'file:///svn/target/trunk'
572 ui.status("options: %s", str(options), level=ui.DEBUG, color='GREEN')
573
574 # Make sure that both the source and target URL's are valid
575 source_info = svnclient.get_svn_info(source_url)
576 assert is_child_path(source_url, source_info['repos_url'])
577 target_info = svnclient.get_svn_info(target_url)
578 assert is_child_path(target_url, target_info['repos_url'])
579
580 # Init global vars
581 global source_repos_url,source_base,source_repos_uuid
582 source_repos_url = source_info['repos_url'] # e.g. 'http://server/svn/source'
583 source_base = source_url[len(source_repos_url):] # e.g. '/trunk'
584 source_repos_uuid = source_info['repos_uuid']
585
586 # Init start and end revision
587 source_start_rev = svnclient.get_svn_rev(source_repos_url, options.svn_rev_start if options.svn_rev_start else 1)
588 source_end_rev = svnclient.get_svn_rev(source_repos_url, options.svn_rev_end if options.svn_rev_end else "HEAD")
589
590 target_end_rev = target_info['revision'] # Last revision # in the target repo
591 wc_target = os.path.abspath('_wc_target')
592 num_entries_proc = 0
593 commit_count = 0
594 source_rev = None
595 target_rev = None
596
597 # Check out a working copy of target_url if needed
598 wc_exists = os.path.exists(wc_target)
599 if wc_exists and not options.cont_from_break:
600 shutil.rmtree(wc_target)
601 wc_exists = False
602 if not wc_exists:
603 ui.status("Checking-out _wc_target...", level=ui.VERBOSE)
604 svnclient.svn_checkout(target_url, wc_target)
605 os.chdir(wc_target)
606
607 if not options.cont_from_break:
608 # TODO: Warn user if trying to start (non-continue) into a non-empty target path?
609 # Get the first log entry at/after source_start_rev, which is where
610 # we'll do the initial import from.
611 it_log_start = svnclient.iter_svn_log_entries(source_url, source_start_rev, source_end_rev, get_changed_paths=False)
612 for source_start_log in it_log_start:
613 break
614 if not source_start_log:
615 raise InternalError("Unable to find any matching revisions between %s:%s in source_url: %s" % \
616 (source_start_rev, source_end_rev, source_url))
617
618 # This is the revision we will start from for source_url
619 source_start_rev = source_rev = int(source_start_log['revision'])
620 ui.status("Starting at source revision %s.", source_start_rev, level=ui.VERBOSE)
621
622 # For the initial commit to the target URL, export all the contents from
623 # the source URL at the start-revision.
624 paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)])
625 if len(paths)>1:
626 disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev))
627 ui.status("(Initial import)", level=ui.VERBOSE)
628 paths = paths.strip("\n").split("\n")
629 for path_raw in paths:
630 # For each top-level file/folder...
631 if not path_raw:
632 continue
633 # Directories have a trailing slash in the "svn list" output
634 path_is_dir = True if path_raw[-1] == "/" else False
635 path = path_raw.rstrip('/') if path_is_dir else path_raw
636 if path_is_dir and not os.path.exists(path):
637 os.makedirs(path)
638 ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path, level=ui.VERBOSE)
639 run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path+"@"+str(source_rev), path])
640 run_svn(["add", path])
641 num_entries_proc += 1
642 target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's
643 target_rev = commit_from_svn_log_entry(source_start_log, options, target_revprops=target_revprops)
644 if target_rev:
645 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
646 set_rev_map(source_rev, target_rev)
647 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
648 run_svn(["update"])
649 commit_count += 1
650 else:
651 # Re-build the rev_map based on any already-replayed history in target_url
652 build_rev_map(target_url, target_end_rev, source_info)
653 if not rev_map:
654 raise RuntimeError("Called with continue-mode, but no already-replayed history found in target repo: %s" % target_url)
655 source_start_rev = int(max(rev_map, key=rev_map.get))
656 assert source_start_rev
657 ui.status("Continuing from source revision %s.", source_start_rev, level=ui.VERBOSE)
658
659 svn_vers_t = svnclient.get_svn_client_version()
660 svn_vers = float(".".join(map(str, svn_vers_t[0:2])))
661
662 # Load SVN log starting from source_start_rev + 1
663 it_log_entries = svnclient.iter_svn_log_entries(source_url, source_start_rev+1, source_end_rev, get_revprops=True) if source_start_rev < source_end_rev else []
664 source_rev = None
665
666 try:
667 for log_entry in it_log_entries:
668 if options.entries_proc_limit:
669 if num_entries_proc >= options.entries_proc_limit:
670 break
671 # Replay this revision from source_url into target_url
672 disp_svn_log_summary(log_entry)
673 source_rev = log_entry['revision']
674 # Process all the changed-paths in this log entry
675 commit_paths = []
676 process_svn_log_entry(log_entry, options, commit_paths)
677 num_entries_proc += 1
678 # Commit any changes made to _wc_target
679 target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's
680 target_rev = commit_from_svn_log_entry(log_entry, options, commit_paths, target_revprops=target_revprops)
681 if target_rev:
682 # Update rev_map, mapping table of source-repo rev # -> target-repo rev #
683 source_rev = log_entry['revision']
684 set_rev_map(source_rev, target_rev)
685 # Update our target working-copy, to ensure everything says it's at the new HEAD revision
686 run_svn(["update"])
687 commit_count += 1
688 # Run "svn cleanup" every 100 commits if SVN 1.7+, to clean-up orphaned ".svn/pristines/*"
689 if svn_vers >= 1.7 and (commit_count % 100 == 0):
690 run_svn(["cleanup"])
691 if not source_rev:
692 # If there were no new source_url revisions to process, init source_rev
693 # for the "finally" message below.
694 source_rev = source_end_rev
695
696 except KeyboardInterrupt:
697 print "\nStopped by user."
698 print "\nCleaning-up..."
699 run_svn(["cleanup"])
700 full_svn_revert()
701 except:
702 print "\nCommand failed with following error:\n"
703 traceback.print_exc()
704 print "\nCleaning-up..."
705 run_svn(["cleanup"])
706 print run_svn(["status"])
707 full_svn_revert()
708 finally:
709 print "\nFinished at source revision %s%s." % (source_rev, " (dry-run)" if options.dry_run else "")
710
711 def main():
712 # Defined as entry point. Must be callable without arguments.
713 usage = "Usage: %prog [OPTIONS] source_url target_url"
714 description = """\
715 Replicate (replay) history from one SVN repository to another. Maintain
716 logical ancestry wherever possible, so that 'svn log' on the replayed
717 repo will correctly follow file/folder renames.
718
719 == Examples ==
720 Create a copy of only /trunk from source repo, starting at r5000
721 $ svnadmin create /svn/target
722 $ svn mkdir -m 'Add trunk' file:///svn/target/trunk
723 $ svn2svn -av -r 5000 http://server/source/trunk file:///svn/target/trunk
724 1. The target_url will be checked-out to ./_wc_target
725 2. The first commit to http://server/source/trunk at/after r5000 will be
726 exported & added into _wc_target
727 3. All revisions affecting http://server/source/trunk (starting at r5000)
728 will be replayed to _wc_target. Any add/copy/move/replaces that are
729 copy-from'd some path outside of /trunk (e.g. files renamed on a
730 /branch and branch was merged into /trunk) will correctly maintain
731 logical ancestry where possible.
732
733 Use continue-mode (-c) to pick-up where the last run left-off
734 $ svn2svn -avc http://server/source/trunk file:///svn/target/trunk
735 1. The target_url will be checked-out to ./_wc_target, if not already
736 checked-out
737 2. All new revisions affecting http://server/source/trunk starting from
738 the last replayed revision to file:///svn/target/trunk (based on the
739 svn2svn:* revprops) will be replayed to _wc_target, maintaining all
740 logical ancestry where possible."""
741 parser = optparse.OptionParser(usage, description=description,
742 formatter=HelpFormatter(), version="%prog "+str(full_version))
743 #parser.remove_option("--help")
744 #parser.add_option("-h", "--help", dest="show_help", action="store_true",
745 # help="show this help message and exit")
746 parser.add_option("-r", "--revision", type="string", dest="svn_rev", metavar="ARG",
747 help="revision range to replay from source_url\n" + \
748 "A revision argument can be one of:\n" + \
749 " START start rev # (end will be 'HEAD')\n" + \
750 " START:END start and ending rev #'s\n" + \
751 "(Any revision # formats which SVN understands\n" + \
752 " are supported, e.g. 'HEAD', '{2010-01-31}', etc.)")
753 parser.add_option("-a", "--keep-author", action="store_true", dest="keep_author", default=False,
754 help="maintain original 'Author' info from source repo")
755 parser.add_option("-c", "--continue", action="store_true", dest="cont_from_break",
756 help="continue from previous break")
757 parser.add_option("-l", "--limit", type="int", dest="entries_proc_limit", metavar="NUM",
758 help="maximum number of log entries to process")
759 parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
760 help="try processing next log entry but don't commit changes to "
761 "target working-copy (forces --limit=1)")
762 parser.add_option("-v", "--verbose", dest="verbosity", action="count", default=1,
763 help="enable additional output (use -vv or -vvv for more)")
764 parser.add_option("--debug", dest="verbosity", const=ui.DEBUG, action="store_const",
765 help="enable debugging output (same as -vvv)")
766 options, args = parser.parse_args()
767 if len(args) != 2:
768 parser.error("incorrect number of arguments")
769 if options.verbosity < 10:
770 # Expand multiple "-v" arguments to a real ui._level value
771 options.verbosity *= 10
772 if options.dry_run:
773 # When in dry-run mode, only try to process the next log_entry
774 options.entries_proc_limit = 1
775 options.svn_rev_start = None
776 options.svn_rev_end = None
777 if options.svn_rev:
778 rev = options.svn_rev.split(":")
779 options.svn_rev_start = rev[0] if len(rev)>0 else None
780 options.svn_rev_end = rev[1] if len(rev)>1 else None
781 ui.update_config(options)
782 return real_main(options, args)
783
784
785 if __name__ == "__main__":
786 sys.exit(main() or 0)