From 027750c831be2e2aca12a5ae739e397a596a65db Mon Sep 17 00:00:00 2001 From: Tony Duckles Date: Wed, 8 Feb 2012 21:33:16 -0600 Subject: [PATCH] Support SVN versioned property updating --- svn2svn/run/svn2svn.py | 116 ++++++++++++++++++++++++++------------ svn2svn/shell.py | 2 +- svn2svn/svnclient.py | 59 +++++++++++++++++++ tests/make-ref-repo.sh | 11 ++++ tests/make-replay-repo.sh | 2 +- 5 files changed, 151 insertions(+), 39 deletions(-) diff --git a/svn2svn/run/svn2svn.py b/svn2svn/run/svn2svn.py index 0a7d85f..48b284e 100644 --- a/svn2svn/run/svn2svn.py +++ b/svn2svn/run/svn2svn.py @@ -105,6 +105,27 @@ def gen_tracking_revprops(source_rev): {'name':'svn2svn:source_rev', 'value':source_rev}] return revprops +def sync_svn_props(source_url, source_rev, path_offset): + """ + Carry-forward any unversioned properties from the source repo to the + target WC. + """ + source_props = svnclient.get_all_props(source_url+"/"+path_offset, source_rev) + target_props = svnclient.get_all_props(path_offset) + if 'svn:mergeinfo' in source_props: + # Never carry-forward "svn:mergeinfo" + del source_props['svn:mergeinfo'] + for prop in target_props: + if prop not in source_props: + # Remove any properties which exist in target but not source + run_svn(["propdel", prop, path_offset]) + for prop in source_props: + if prop not in target_props or \ + source_props[prop] != target_props[prop]: + # Set/update any properties which exist in source but not target or + # whose value differs between source vs. target. + run_svn(["propset", prop, source_props[prop], path_offset]) + def in_svn(p, require_in_repo=False, prefix=""): """ Check if a given file/folder is being tracked by Subversion. @@ -396,9 +417,15 @@ def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom ui.status(prefix + ">> do_svn_add: pre-copy: local path already exists: %s", path_offset, level=ui.DEBUG, color='GREEN') run_svn(["remove", "--force", path_offset]) run_svn(["copy", "-r", tgt_rev, target_url+"/"+copyfrom_offset+"@"+str(tgt_rev), path_offset]) - # Export the final version of this file/folder from the source repo, to make - # sure we're up-to-date. - add_path(export_paths, path_offset) + if is_dir: + # Export the final verison of all files in this folder. + add_path(export_paths, path_offset) + else: + # Export the final verison of this file. + run_svn(["export", "--force", "-r", source_rev, + source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset]) + # Copy SVN properties from source repo + sync_svn_props(source_url, source_rev, path_offset) else: ui.status(prefix + ">> do_svn_add: Skipped 'svn copy': %s", path_offset, level=ui.DEBUG, color='GREEN') else: @@ -421,7 +448,8 @@ def do_svn_add(path_offset, source_rev, parent_copyfrom_path="", parent_copyfrom source_repos_url+source_base+"/"+path_offset+"@"+str(source_rev), path_offset]) # If not already under version-control, then "svn add" this file/folder. run_svn(["add", "--parents", path_offset]) - # TODO: Need to copy SVN properties from source repos + # Copy SVN properties from source repo + sync_svn_props(source_url, source_rev, path_offset) if is_dir: # For any folders that we process, process any child contents, so that we correctly # replay copies/replaces/etc. @@ -536,18 +564,18 @@ def process_svn_log_entry(log_entry, options, commit_paths, prefix = ""): # Need to use in_svn here to handle cases where client committed the parent # folder and each indiv sub-folder. run_svn(["add", "--parents", path_offset]) - # TODO: Need to copy SVN properties from source repos + # Copy SVN properties from source repo + sync_svn_props(source_url, source_rev, path_offset) elif action == 'D': run_svn(["remove", "--force", path_offset]) elif action == 'M': - # TODO: Is "svn merge -c" correct here? Should this just be an "svn export" plus - # proplist updating? - out = run_svn(["merge", "-c", source_rev, "--non-recursive", - "--non-interactive", "--accept=theirs-full", - source_url+"/"+path_offset+"@"+str(source_rev), path_offset]) - # TODO: If d['props'] == 'modified', then run code to clean-up/purge any newly-modified props? + if path_is_file: + run_svn(["export", "--force", "-N" , "-r", source_rev, + source_url+"/"+path_offset+"@"+str(source_rev), path_offset]) + # Update SVN properties based on source repo + sync_svn_props(source_url, source_rev, path_offset) else: raise InternalError("Internal Error: process_svn_log_entry: Unhandled 'action' value: '%s'" @@ -624,32 +652,46 @@ def real_main(options, args): # For the initial commit to the target URL, export all the contents from # the source URL at the start-revision. - paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)]) - if len(paths)>1: - disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev)) - ui.status("(Initial import)", level=ui.VERBOSE) - paths = paths.strip("\n").split("\n") - for path_raw in paths: - # For each top-level file/folder... - if not path_raw: - continue - # Directories have a trailing slash in the "svn list" output - path_is_dir = True if path_raw[-1] == "/" else False - path = path_raw.rstrip('/') if path_is_dir else path_raw - if path_is_dir and not os.path.exists(path): - os.makedirs(path) - ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path, level=ui.VERBOSE) - run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path+"@"+str(source_rev), path]) - run_svn(["add", path]) - num_entries_proc += 1 - target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's - target_rev = commit_from_svn_log_entry(source_start_log, options, target_revprops=target_revprops) - if target_rev: - # Update rev_map, mapping table of source-repo rev # -> target-repo rev # - set_rev_map(source_rev, target_rev) - # Update our target working-copy, to ensure everything says it's at the new HEAD revision - run_svn(["update"]) - commit_count += 1 + disp_svn_log_summary(svnclient.get_one_svn_log_entry(source_url, source_rev, source_rev)) + ui.status("(Initial import)", level=ui.VERBOSE) + # Export and add file-contents from source_url@source_start_rev + top_paths = run_svn(["list", "-r", source_rev, source_url+"@"+str(source_rev)]) + top_paths = top_paths.strip("\n").split("\n") + for path in top_paths: + # For each top-level file/folder... + if not path: + continue + # Directories have a trailing slash in the "svn list" output + path_is_dir = True if path[-1] == "/" else False + path_offset = path.rstrip('/') if path_is_dir else path + if in_svn(path_offset, prefix=" "): + raise InternalError("Cannot replay history on top of pre-existing structure: %s" % source_url+"/"+path_offset) + if path_is_dir and not os.path.exists(path_offset): + os.makedirs(path_offset) + run_svn(["export", "--force", "-r" , source_rev, source_url+"/"+path_offset+"@"+str(source_rev), path_offset]) + run_svn(["add", path_offset]) + # Update any properties on the newly added content + paths = run_svn(["list", "--recursive", "-r", source_rev, source_url+"@"+str(source_rev)]) + paths = paths.strip("\n").split("\n") + sync_svn_props(source_url, source_rev, "") + for path in paths: + if not path: + continue + # Directories have a trailing slash in the "svn list" output + path_is_dir = True if path[-1] == "/" else False + path_offset = path.rstrip('/') if path_is_dir else path + ui.status(" A %s", source_url[len(source_repos_url):]+"/"+path_offset, level=ui.VERBOSE) + sync_svn_props(source_url, source_rev, path_offset) + # Commit the initial import + num_entries_proc += 1 + target_revprops = gen_tracking_revprops(source_rev) # Build source-tracking revprop's + target_rev = commit_from_svn_log_entry(source_start_log, options, target_revprops=target_revprops) + if target_rev: + # Update rev_map, mapping table of source-repo rev # -> target-repo rev # + set_rev_map(source_rev, target_rev) + # Update our target working-copy, to ensure everything says it's at the new HEAD revision + run_svn(["update"]) + commit_count += 1 else: # Re-build the rev_map based on any already-replayed history in target_url build_rev_map(target_url, target_end_rev, source_info) diff --git a/svn2svn/shell.py b/svn2svn/shell.py index 988dba6..3dfecbf 100644 --- a/svn2svn/shell.py +++ b/svn2svn/shell.py @@ -90,7 +90,7 @@ def shell_quote(s): def _run_raw_command(cmd, args, fail_if_stderr=False, no_fail=False): cmd_string = "%s %s" % (cmd, " ".join(map(shell_quote, args))) color = 'BLUE_B' - if cmd == 'svn' and args[0] in ['status', 'st', 'log', 'info', 'list', 'propset', 'update', 'up', 'cleanup', 'revert']: + if cmd == 'svn' and args[0] in ['status', 'st', 'log', 'info', 'list', 'proplist', 'propget', 'update', 'up', 'cleanup', 'revert']: # Show status-only commands (commands which make no changes to WC) in dim-blue color = 'BLUE' ui.status("$ %s", cmd_string, level=ui.EXTRA, color=color) diff --git a/svn2svn/svnclient.py b/svn2svn/svnclient.py index 7f2a398..858e99c 100644 --- a/svn2svn/svnclient.py +++ b/svn2svn/svnclient.py @@ -354,3 +354,62 @@ def get_svn_client_version(): _svn_client_version = tuple(map(int, [x for x in raw.split('.') if x.isdigit()])) return _svn_client_version + + +def parse_svn_propget_xml(xml_string): + """ + Parse the XML output from an "svn propget" command and extract useful + information as a dict. + """ + d = {} + xml_string = strip_forbidden_xml_chars(xml_string) + tree = ET.fromstring(xml_string) + prop = tree.find('.//property') + d['name'] = prop.get('name') + d['value'] = prop is not None and prop.text and prop.text.replace('\r\n', '\n').replace('\n\r', '\n').replace('\r', '\n') or "" + return d + +def parse_svn_proplist_xml(xml_string): + """ + Parse the XML output from an "svn proplist" command and extract list + of property-names. + """ + l = [] + xml_string = strip_forbidden_xml_chars(xml_string) + tree = ET.fromstring(xml_string) + for prop in tree.findall('.//property'): + l.append(prop.get('name')) + return l + +def get_prop_value(svn_url_or_wc, prop_name, rev_number=None): + """ + Get the value of a versioned property for the given path. + """ + args = ['propget', '--xml'] + url = str(svn_url_or_wc) + if rev_number: + args += ['-r', rev_number] + if not "@" in svn_url_or_wc: + url = "%s@%s" % (svn_url_or_wc, str(rev_number)) + args += [prop_name, url] + xml_string = run_svn(args) + return parse_svn_propget_xml(xml_string) + +def get_all_props(svn_url_or_wc, rev_number=None): + """ + Get the values of all versioned properties for the given path. + """ + l = {} + args = ['proplist', '--xml'] + url = str(svn_url_or_wc) + if rev_number: + args += ['-r', rev_number] + if not "@" in svn_url_or_wc: + url = "%s@%s" % (svn_url_or_wc, str(rev_number)) + args += [url] + xml_string = run_svn(args) + props = parse_svn_proplist_xml(xml_string) + for prop_name in props: + d = get_prop_value(svn_url_or_wc, prop_name, rev_number) + l[d['name']] = d['value'] + return l diff --git a/tests/make-ref-repo.sh b/tests/make-ref-repo.sh index 004f1a5..627791a 100755 --- a/tests/make-ref-repo.sh +++ b/tests/make-ref-repo.sh @@ -43,10 +43,13 @@ mkdir -p $WC/Module/ProjectA echo "Module/ProjectA/FileA1.txt (Initial)" >> $WC/Module/ProjectA/FileA1.txt echo "Module/ProjectA/FileA2.txt (Initial)" >> $WC/Module/ProjectA/FileA2.txt svn -q add $WC/Module +svn propset -q desc "FileA1.txt" $WC/Module/ProjectA/FileA1.txt +svn propset -q desc "FileA2.txt" $WC/Module/ProjectA/FileA2.txt svn_commit "Initial population" # Test #1: Add new file # * Test simple copy-from branch +# * Test propset BRANCH="$REPOURL/branches/test1" svn copy -q -m "Create branch" $TRUNK $BRANCH svn switch -q $BRANCH @@ -54,6 +57,7 @@ show_last_commit mkdir -p $WC/Module/ProjectB echo "Module/ProjectB/FileB1.txt (Test 1)" >> $WC/Module/ProjectB/FileB1.txt svn add -q $WC/Module/ProjectB +svn propset -q filename FileB1.txt $WC/Module/ProjectB/FileB1.txt svn_commit "Test 1: Add Module/ProjectB" svn switch -q $TRUNK svn merge -q $BRANCH @@ -68,6 +72,7 @@ svn_commit "" # Test #2: Rename files # * Test rename support # * Test committing rename in two different branch commits: first deletion, then add +# * Test props from copy-from file carry forward to target BRANCH="$REPOURL/branches/test2" svn copy -q -m "Create branch" $TRUNK $BRANCH svn switch -q $BRANCH @@ -86,6 +91,7 @@ svn copy -q -m "Create branch" $TRUNK $BRANCH svn switch -q $BRANCH show_last_commit echo "Module/ProjectB/FileB2.txt (Test 3)" >> $WC/Module/ProjectB/FileB2.txt +svn propset -q filename FileB2.txt $WC/Module/ProjectB/FileB2.txt svn_commit "Test 3: Verify Module/ProjectB/FileB2.txt" svn switch -q $TRUNK svn merge -q $BRANCH @@ -108,6 +114,7 @@ svn_commit "Test 4: Replace Module/ProjectA/FileA1.txt" # Test #5: Rename files + folders # * Test rename support # * Create complex find-ancestors case, where files are renamed within a renamed folder on a branch +# * Test propset and propdel BRANCH="$REPOURL/branches/test5" svn copy -q -m "Create branch" $TRUNK $BRANCH svn switch -q $BRANCH @@ -115,7 +122,11 @@ show_last_commit svn mv -q Module/ProjectB Module/ProjectC svn mv -q Module/ProjectC/FileB1.txt Module/ProjectC/FileC1.txt echo "Module/ProjectC/FileC1.txt (Test 5)" >> $WC/Module/ProjectC/FileC1.txt +svn propdel -q filename $WC/Module/ProjectC/FileC1.txt +svn propset -q desc 'This is a long string +broken on two lines...' $WC/Module/ProjectC/FileC1.txt svn mv -q Module/ProjectC/FileB2.txt Module/ProjectC/FileC2.txt +svn propset -q filename FileC2.txt $WC/Module/ProjectC/FileC2.txt echo "Module/ProjectC/FileC2.txt (Test 5)" >> $WC/Module/ProjectC/FileC2.txt svn_commit "Test 5: Rename Module/ProjectB -> Module/ProjectC" svn switch -q $TRUNK diff --git a/tests/make-replay-repo.sh b/tests/make-replay-repo.sh index 491560d..7843817 100755 --- a/tests/make-replay-repo.sh +++ b/tests/make-replay-repo.sh @@ -15,7 +15,7 @@ svnadmin create $REPO echo "" ## svn2svn / -#../svn2svn.py -a -v file://$PWD/_repo_ref file://$PWD/_repo_replay +#../svn2svn.py -a $1 file://$PWD/_repo_ref file://$PWD/_repo_replay # svn2svn /trunk svn mkdir -q -m "Add /trunk" $REPOURL/trunk -- 2.43.0