Backup from archive.org for http://unix-girl.com/cc2svn/convert_clearcase.pl

convert_clearcase.pl

#!/usr/bin/perl -w
#
# Script to convert ClearCase historical information to Subversion.
#
# See pod documentation on the bottom of the script for details.
#
# Version 1.0.2 
#
# Revision history
#==================================================
# 1.0.3 b25
#       i.   Option to skip subversion checkout process.
#       ii.  Better handling of $ in file names.
#
# 1.0.2 b21
#       i.   Script will sleep when a configured (--sleepfile) sleepfile exists
#            and resume its operation once the file is removed. 
#       ii.  Unlink files even on first run -- just in case.
#       iii. Escape file names & paths for those <explicit deleted>  who put spaces in them. 
#       iv.  Better handling for spaces in file & path names. 
#       v.   Parameter to override svn url
#
# 1.0.1 b17
#       i.  Change timestamp of each version to original check-in date + random time
#           to get around subversion not seeing files as different when timestamp
#           has not changed (problem shows up when script runs too fast). 
#       ii. Additional logging to help debug problems.
#
#
# To do:
# 1. Fix handling of symlinks.
#

use strict;
use Cwd;
use DB_File; 
use Fcntl qw(:DEFAULT);
use File::Basename;
use File::Find;
use File::Path;
use IO::Handle; 
use Time::HiRes qw(sleep); 

#==================================================
# Things to configure
#==================================================

my $cc_command       	= "cleartool";
my $cc_path          	= "";
my $cc_sub_path      	= "";
my $cc_branch        	= "main"; 
my $skip_directories 	= ""; 
my $svn_command      	= "svn";
my $svn_server       	= "localhost";
my $svn_connection_type = "svn";
my $svn_repository      = "repository"; 
my $svn_repository_path = "/home/$svn_repository";
my $svn_user            =  "svnadmin";
my $svn_password        = "foo"; 
my $test_mode           = 0; 
my $log                 = "log";
my $log_level           = 1; 
my $working_dir         = "";
my $date_limit          = 0; 
my $count_limit         = 0; 
my $lowercase_usernames = 1; 
my $user_file           = "userfile";
my $use_real_users      = 1; 
my $user_password       = "foo"; 
my $retry_commits       = 10; 
my $retry_in_seconds    = 60; 
my $exit_file           = "convert_exit";
my $db_file             = "convertdb"; 
my $sleep_file          = "/home/ktrapszo/convert_sleep"; 
my $svn_override_url    = undef; 
my $no_checkout         = 0; 

#==================================================
# Nothing to configure below
#==================================================

# Force bash
#
$ENV{'SHELL'} = '/bin/bash';

# Disable buffering
#
$| = 1; 
# dump leading part of basename path
#
$0 =~ s|.*/||;  

foreach ( @ARGV )
{
    if ( m/^--help/i )
    {
	print STDERR <<EOF;
usage: $0 [options]
	  
  --help           Show this brief help listing
  --testmode       Show what we would do, but don not actually do it 
  --userealusers   Use real users when checking into subversion
  --lcaseusers     Convert usernames to lower case 
  --logfile=F      Output logfile is F (default \"$log\")
  --loglevel=N     0=none 1=Summary 2=More info 3=Lots-and-lots (default=$log_level)
  --cccommand=C    Command to invoke clearcase (default \"$cc_command\")
  --ccpath=P       Use P as the path for the ClearCase tree to convert (default \"$cc_path\")
  --ccsubpath=P    Use P as subpath appended to above path, read documentation for details (default \"$cc_sub_path\")
  --ccbranch=B     branch name to convert  (default \"$cc_branch\")
  --skipdir=D      Skip directories D, list separated by pipe (default \"$skip_directories\")
  --svncommand=C   Command to invoke subversion (default \"$svn_command\")
  --svnserver=S    Subversion server hostname/IP (default \"$svn_server\")
  --svnctype=T     Connection type (svn, svn+ssh, file, http) (default \"$svn_connection_type\")
  --svnrepos=R     Subversion repository = R (default \"$svn_repository\")
  --svnrepospath=P Path to Subversion repository (default \"$svn_repository_path\")
  --svnuser=U      Subversion user = U (default \"$svn_user\")
  --svnpass=P      Subversion password = P (default \"$svn_password\")
  --svnurl=U       Overwrite the script generated svn url
  --workingdir=D   Directory for doing our work in (default current dir)
  --datelimit=D    Processing starting at this date (dd-mm-yyyy) (default none)
  --countlimit=N   Process only <N> events (default none)
  --userfile=F     Dump user information to file <F> (default \"$user_file\")
  --commits=N      Retry commits up to <N> times (default $retry_commits)
  --commitsleep=N  Sleep <N> seconds between commit retries (default $retry_in_seconds)
  --exitfile=F     If this file is seen, exit gracefully (default \"$exit_file\")
  --dbfile=F       DBFile to store conversion history (default \"$db_file\")
  --sleepfile=F    While this file exists the script will sleep (default \"$sleep_file\")
  --nocheckout     Instead of checking out the svn repository run an update default is checkout
  
EOF
exit 0;
    }
    elsif ( m/^--testmode$/ )
    {
	$test_mode = 1;
    }
    elsif (m/^--userealusers$/)
    {
	$use_real_users = 1; 
    }
    elsif ( m/^--lcaseusers$/ )
    {
	$lowercase_usernames = 1;
    }
    elsif ( m/^--logfile=(.+)$/ )
    {
	$log = $1;
    }
    elsif ( m/^--loglevel=(\d+)$/ )
    {
	$log_level = $1;
    }
    elsif (m/^--cccommand=(.+)$/)
    {
	$cc_command=$1;
    }
    elsif ( m/^--ccpath=(.+)$/ )
    {
	$cc_path = $1;
    }
    elsif ( m/^--ccsubpath=(.+)$/)
    {
	$cc_sub_path=$1;
    }
    elsif ( m/^--ccbranch=(.+)$/ )
    {
	$cc_branch = $1;
    }
    elsif (m/^--skipdir=(.+)$/)
    {
	$skip_directories=$1;
    }
    elsif(m/^--svncommand=(.+)$/)
    {
	$svn_command=$1;
    }
    elsif ( m/^--svnserver=(.+)$/ )
    {
	$svn_server = $1;
    }
    elsif ( m/^--svnctype=(.+)$/ )
    {
	$svn_connection_type = $1;
    }
    elsif ( m/^--svnrepos=(.+)$/ )
    {
	$svn_repository = $1;
	$svn_repository_path = "/home/$svn_repository"; 
    }
    elsif(m/^--svnrepospath=(.+)$/)
    {
	$svn_repository_path = $1;
    }
    elsif ( m/^--svnuser=(.+)$/ )
    {
	$svn_user = $1;
    }
    elsif ( m/^--svnpass=(.+)$/ )
    {
	$svn_password = $1;
    }
    elsif(m/^--workingdir=(.+)$/)
    {
	$working_dir = $1;
    }
    elsif ( m/^--datelimit=(.+)$/ )
    {
	$date_limit = $1;
    }
    elsif ( m/^--countlimit=(\d+)$/ )
    {
	$count_limit = $1;
    }
    elsif ( m/^--userfile=(.+)$/ )
    {
	$user_file = $1;
    }
    elsif ( m/^--commits=(\d+)$/ )
    {
	$retry_commits = $1;
    }
    elsif ( m/^--commitsleep=(\d+)$/ )
    {
	$retry_in_seconds = $1;
    }
    elsif ( m/^--exitfile=(.+)$/ )
    {
	$exit_file = $1;
    }
    elsif(m/^--dbfile=(.+)$/)
    {
	$db_file = $1;
    }
    elsif(m/^--sleepfile=(.+)$/)
    {
	$sleep_file = $1;
    }
    elsif(m/^--svnurl=(.+)$/)
    {
	$svn_override_url = $1;
    }
    elsif(m/^--nocheckout/)
    {
	$no_checkout = 1; 
    }
    else
    {
	die "ERROR: {$_} is bad cmdline param (try --help)\n";
    }
}

# ------------------------------------------------------------------------
# SANITY CHECKING ON PARAMS - make sure we have everything we need
#

die "ERROR: missing --ccpath=PATH parameter\n"          if not $cc_path;
die "ERROR: --ccpath=$cc_path directory missing\n"      if not -e $cc_path;
die "ERROR: --ccpath=$cc_path is not a directory\n"     if not -d $cc_path;

die "ERROR: missing --ccbranch=BRANCH parameter\n"      if not $cc_branch;


die "ERROR: missing --svnserver=SERVER param\n"         if not $svn_server;
die "ERROR: missing --svnctype=CTYPE param\n"           if not $svn_connection_type;

die "ERROR: missing --svnuser=U param\n"        	if not $svn_user;
die "ERROR: missing --svnpass=PASS param\n"     	if not $svn_password;
die "ERROR: missing --svnserver=S param\n"      	if not $svn_server;
die "ERROR: missing --svnrepos=REPOS param\n"   	if not $svn_repository;


$cc_path     =~ s|/+$||;        # dump trailnig path
$cc_sub_path =~ s|/+$||;        # dump trailing slash



my $now                 = scalar(localtime); 
my $need_to_add_paths   = 0; 
my $num_directories     = 0;
my $num_files           = 0; 
my $rev_high_mark       = 0; 
my $rev_high_file       = ""; 
my %users_list          = (); 
my %ver_hash            = (); 
my $commit_retries_left = $retry_commits; 
my $svn_url             = "$svn_connection_type://$svn_server$svn_repository_path";
my $g_hour = 0; 
my $g_min  = 0; 

if($svn_override_url)
{
    $svn_url = $svn_override_url;
}

if(!$working_dir)
{
    $working_dir = getcwd;
}

if($log_level > 0)
{
    open (LOG,  ">>$working_dir/$log") or die "Unable to open log file ($log): $!\n";
    
    # Disable buffering for the log
    #
    LOG->autoflush(1); 

    print LOG "========================= $now =========================\n";
    print LOG "Starting conversion\n";

    if($log_level > 1)
    {
	print LOG "Using settings:\n";
	print LOG "\tClearCase Path             = $cc_path\n";
	print LOG "\tClearCase Sub Path         = $cc_sub_path\n";
	print LOG "\tSubversion command         = $svn_command\n";
	print LOG "\tSubversion host            = $svn_server\n";
	print LOG "\tSubversion connection type = $svn_connection_type\n";
	print LOG "\tSubversion Repository      = $svn_repository\n";
	print LOG "\tSubversion Repository path = $svn_repository_path\n"; 
	print LOG "\tSubversion URL             = $svn_url\n";
	print LOG "\tSubversion username        = $svn_user\n";
	print LOG "\tWorking Direcotry          = $working_dir\n"; 
    }

}

checkForExit(); 

tie(%ver_hash, "DB_File", "$working_dir/$db_file", O_RDWR|O_CREAT) or die "Cannot tie $db_file: $!\n"; 

if(!$no_checkout)
{
   # First step - checkout the repository from subversion
   #
    print "Checking out subversion repository...";
    my $result = checkoutSvnRepository();
    print "\nResut: $result\n"; 
    
    if($result !~ m/Checked out revision/i)
    {
	my $msg = "Checkout unsucessful, exiting.\n";
	print $msg;
	
	if($log_level > 0) 
	{
	    print LOG "$msg" ;
	}
	doExit(); 
    }
    
    if($log_level > 2)
    {
	print LOG "Checkout result: $result\n";
    }
}


my $search_path  = $cc_path;
my $svn_top_path = $working_dir  . "/" . $svn_repository; 
my $new_path     = $svn_top_path;

if($cc_sub_path)
{
    $new_path    .= "/" . $cc_sub_path; 
    $search_path .= "/" . $cc_sub_path;
}



# First we need to create all the paths necessary
# in Subversion
#
print "Processing paths...\n"; 
find(\&processPaths, $search_path); 

# No directories found, means nothing to convert
#
if(!$num_directories)
{
    print "Couldn't find directories to process, check your settings\n"; 
    doExit(); 
}

# Add any new paths to Subversion  if necessary - this
# will happen if we had to make any new directories.
#
if($need_to_add_paths)
{
    my $path_to_add = getTopLevelPath($new_path); 
    print "Adding new path to subversion: $path_to_add\n";
    addToSVN($path_to_add); 
    print "Committing new directories to Subversion...\n"; 
    commitToSVN("New path added: $path_to_add"); 
}

# Now that we have all the paths we need, do the actual conversion
# 
find(\&processHistory, $search_path); 

print "$num_directories directories and $num_files files converted\n"; 

doExit(); 


# Checkout subversion repository. If this
# step fails, we log and exit with an error
#
sub checkoutSvnRepository
{
    my $command = "$svn_command checkout";
    my $rval    = ""; 

    if($svn_user)
    {
	$command .= " --username " . qq{"$svn_user"};
	$command .= " --password " . qq{"$svn_password"};
    }
    
    $command .= " $svn_url 2>&1";

    if($log_level > 1)
    {
	$now = scalar(localtime); 
	print LOG "$now: Checking out Subversion repository  with command: \"$command\"\n";
    }
    
    if(!$test_mode)
    {
	$rval = `$command 2>&1`; 
    }
    else
    {
	$rval = "would have executed $command\n"; 
    }
    return $rval; 
}


# Traverses the ClearCase repository and creates any paths
# That are non-existant in Subversion.
#
sub processPaths
{
    
    my $full_path = $File::Find::name;
    my $curr_dir  = $File::Find::dir;
    my $base_name = $_; 
    
    checkForExit(); 

    if(-d)
    {

	if($base_name =~ m/$skip_directories/)
	{
	    $File::Find::prune = 1; 
	}
	else
	{
	    if($log_level > 1) 
	    {
		print LOG "Working in $full_path\n";
	    }
	    checkPath($full_path); 
	    $num_directories++; 
	}
    }
}

# Recursively traverse the clearcase tree
# and convert each version for each clearcase element. 
#
sub processHistory
{
    
    my $full_path = $File::Find::name;
    my $curr_dir  = $File::Find::dir;
    my $base_name = $_; 
    
    checkForExit(); 

    if(-d)
    {
	$now = scalar(localtime); 
	print "\n$now: Working in $full_path...";

	if($log_level > 1) 
	{
	    print LOG "$now: Working in $full_path\n";
	}
	#opendir(DIR, "$base_name"); 

	# Get a list of clear case vob objects in this directory
	#
	my @filenames = `$cc_command ls -short -nxname -vob_only \"$base_name\" 2>&1`;
	#closedir DIR;
	
	
	foreach my $file (@filenames)
	{
	    
	    checkForExit(); 

	    $file = trim($file); 
	    $file =~ s/\$/\\\$/g;

	    # Skip directories
	    #
	    next if (-d "$file");

	    $num_files++; 

	    if($log_level > 2)
	    {
		print LOG "Working with $file\n"; 
	    }

	    my @versions = getVersionHistory($file);

	    # If versions fell outside the date range, get at least the latest
	    # version.
            #
	    if(!@versions)
	    {
		@versions = getVersionHistory($file, 1); 
	    }
	    
	    my $first = 1; 
	    my $lhigh_mark = 0; 

	    for(my $i = $#versions; $i >= 0; $i--)
	    {
		# Before we do anything for each version, let's make
		# sure we're not supposed to exit.
		#
		checkForExit(); 

		my $curr_ref = $versions[$i];
				
		my $version = $curr_ref->{"version"}; 
		my $comment = $curr_ref->{"comment"};
		my $date    = $curr_ref->{"date"};
		my $user    = $curr_ref->{"user"};

		my $new_comment = "$user on $date $comment"; 
		convertVersion($file, $version, $new_comment, $first, $user, $full_path, $date); 
		if($first)
		{
		    $first = 0; 
		}
		
		$users_list{$user} = 1 unless $users_list{$user}; 

		$lhigh_mark++; 
		print "."; 
	    }
	    
	    # Remember which file had the most revisions, for fun statistics.
	    #
	    if($lhigh_mark > $rev_high_mark)
	    {
		$rev_high_mark = $lhigh_mark;
		$rev_high_file = $file; 
	    }
	}
    }
}

# get a list of all versions & history for the file
#
sub getVersionHistory
{
    my $file           = shift; 
    my $override_count = shift; 
    my @rval           = (); 

    my $command = "$cc_command lsh";
    
    if(!$override_count)
    {
	if($count_limit)
	{
	    $command .= " -last $count_limit"; 
	}
	
	if($date_limit)
	{
	    $command .= " -since $date_limit"; 
	}
    }
    else
    {
	$command .= " -last $override_count"; 
    }


    $command .= " \"" . $file . "\""; 
    
    if($log_level > 2)
    {
	print LOG "Executing history with \"$command\"\n";
    }

    my $index      = 0; 
    my $new_record = 0; 

    foreach my $result (`$command 2>&1`)
    {
	$result = trim($result); 
	
	# We're skipping branch creation.
        # and import.. since clearcase stores it as a separate
	# revision anyway so we won't miss it. 
	#
	next if($result =~ m/create branch/i);
	next if($result =~ m/create file element/i); 
	next if($result =~ m/import file element/i); 

	if($result =~ m/create version/i)
	{
	    $new_record = 1;
	    
	    my ($part_one, $part_two) = split(/\s+create\s+version\s+/, $result); 
	    my ($date, $user, $user2) = split(/\s+/, $part_one); 
	    my ($ver) = split(m/\"\s+/, $part_two); 
	    
	    # Dump trailing &| leading " 
	    #
	    $ver =~ s/^\"//;
	    $ver =~ s/\"$//;
	    
	    # escape any $ in the filename
	    #
	    $ver =~ s/\$/\\\$/g;

	    # Some usernames have spaces, yuck
	    #
	    $user .= $user2    if $user2; 
	    $user  = lc($user) if $lowercase_usernames; 

	    # Get the date in a usable format
	    #
	    my ($lday, $lmon, $lyear) = split(/-/, $date);
	   
	    if($lmon)
	    {
		$lmon = "01" if($lmon =~ m/Jan/i);
		$lmon = "02" if($lmon =~ m/Feb/i); 
		$lmon = "03" if($lmon =~ m/Mar/i); 
		$lmon = "04" if($lmon =~ m/Apr/i); 
		$lmon = "05" if($lmon =~ m/May/i); 
		$lmon = "06" if($lmon =~ m/Jun/i); 
		$lmon = "07" if($lmon =~ m/Jul/i); 
		$lmon = "08" if($lmon =~ m/Aug/i); 
		$lmon = "09" if($lmon =~ m/Sep/i); 
		$lmon = "10" if($lmon =~ m/Oct/i); 
		$lmon = "11" if($lmon =~ m/Nov/i); 
		$lmon = "12" if($lmon =~ m/Dec/i); 
	    }

	    if(!$lyear)
	    {
		$lyear   = (localtime(time))[5] + 1900; 
		my $cmon = (localtime(time))[4] + 1; 
		if($lmon > $cmon) 
		{
		    $lyear--;
		}
	    }
	    
	    $date = "$lyear-$lmon-$lday"; 

	    $rval[$index++] = {
		version => $ver,
		date    => $date,
		user    => $user,
		comment => ""
		};
	}
	else
	{
	    next if (!$new_record); 
	    
	    # this is the comment associated with the previous record.
	    #
	    my $curr_ref = $rval[$index-1];
	    $curr_ref->{"comment"} .= trim($result); 
	}
	
	if($log_level > 2)
	{
	    print LOG "\t $result\n"; 
	}
    }

    return @rval; 
}

# This is the actual conversion work.
#
sub convertVersion
{
    my $file     = shift; 
    my $version  = shift;
    my $comment  = shift;
    my $first    = shift; 
    my $user     = shift; 
    my $curr_dir = shift; 
    my $date     = shift; 
    
    $curr_dir =~ s|/[^/]+$||; 
    
    my $addon       = remainder_path($search_path, $curr_dir); 
    my $svn_path    = $new_path . "/" . $addon . "/" . $file; 
    my $version_key = $cc_sub_path . "/";
    $version_key .= $addon . "/" if $addon;
    $version_key .= $version; 


    # Check if the version wasn't already converted!
    #
    if($ver_hash{$version_key})
    {
	my $msg = "$version_key has been converted, skipping\n"; 

	if($log_level > 1)
	{
	    print LOG $msg; 
	}
	# Remove the file anyway, in case we have more versions of it to convert
	#
	my $result = `rm -f \"$svn_path\" 2>&1`; 
	print LOG "$result of remove file\n" if($result and $log_level > 2);
	
    }
    else
    {

	# Remove previous version from path
	#
	my $result = `rm -f \"$svn_path\" 2>&1`; 
	print LOG "$result of remove file\n" if($result and $log_level > 2);

		
	my $command = "$cc_command get  -to \"$svn_path\" \"$version\""; 
	
	$result = `$command 2>&1`;
	print LOG "Get result: $result\n"   if $log_level > 2;
	print LOG "Get command: \"$command\"\n" if $log_level > 2; 

	if($result =~ m/cleartool: Error:/)
	{
	    print LOG "ERROR: encountered during converstion: $result\n"; 
	    print LOG "Info for error: \n";
	    print LOG "\t file: $file\n"; 
	    print LOG "\t version: $version\n";
	    print LOG "\t comment: $comment\n"; 
	    print LOG "\t first: $first\n";
	    print LOG "\t user: $user\n";
	    print LOG "\t curr_dir: $curr_dir\n"; 
	}
	else
	{
	    # Change the date of the file to the original version date. 
	    #
	    my ($year, $mon, $day) = split(/-/, $date); 
	    $date = "$year$mon$day";
	    
	    # Just to make sure we don't run into an identical timestamp
	    # on the same file -svn looks at timestamp to determine if a file 
	    # was changed (stupid)
	    #
	    if ($g_min >= 59)
	    {
		$g_min  = 0; 
		$g_hour++; 
	    }
	    
	    if($g_hour > 23)
	    {
		$g_hour = 0; 
	    }

	    my $lmin  = $g_min++; 
	    my $lhour = $g_hour; 
	    
	    $lmin  = "0$lmin" if($lmin < 10);
	    $lhour = "0$lhour" if($lhour < 10); 
		
	    $date .= "$lhour$lmin"; 


	    $command = "touch -t $date \"$svn_path\""; 
	    $result  = `$command 2>&1`;

	    print LOG "Changing file date with command \"$command\"\n" if $log_level > 2; 

	    if($first)
	    {
		addToSVN($svn_path, 1, $comment, $user, $version_key); 
	    }
	    else
	    {
		$svn_path =~ s|/[^/]+$||; 
		commitToSVN($comment, $svn_path, $user, $version_key); 
	    }
	}
    }
}

# Either ensure a directory we need already exists 
# in subversion repository - create it if it doesn't. 
# 
sub checkPath
{
    my $curr_dir        = shift; 
    my $curr_cc_dir     = shift; 

    my $addon = remainder_path($search_path, $curr_dir); 
    
    my $repository_path = $new_path .  "/" . $addon;

    if( not -d $repository_path)
    {
	eval {  mkpath ("$repository_path", 1, 0766) };
	if($@)
	{
	    print "Unable to create new repository path ($repository_path), exiting: $@" ;
	    doExit(); 
	}
	else
	{
	    $need_to_add_paths = 1; 
	}
	     
    }
}

# Add provided path to Subversion, it will recurse up until it
# hits a working copy.
#
sub getTopLevelPath
{
    my $path      = shift;
    my $rval      = 0; 
    my $prev_path = $path; 

    while($path)
    {
	(my @result = `$svn_command info $path 2>&1`) =~ s/\s+$//;
	foreach my $line(@result)
	{
	    if($line =~ m/Node Kind/i)
	    {
		$rval = $prev_path; 
		last; 
	    }
	}
	
	last if($rval);
	    
	$prev_path = $path; 
	$path      =~ s|/[^/]+$||; # dump tail path
    }
    
    return $rval; 
}


# Add the provided element to Subversion, will commit if asked to.
#
sub addToSVN
{
    my $elem         = shift; 
    my $commit       = shift; 
    my $comment      = shift; 
    my $user         = shift; 
    my $version_key  = shift; 

    my $command = "$svn_command add " . path_escape($elem);
    my $result  = "";
    

    if($test_mode)
    {
	$result = "Would have executed \"$command\""; 
    }
    else
    {
	$result = `$command 2>&1`;
    }
    
    if($log_level > 1)
    {
	print LOG "Executing add to SVN with: \"$command\"\n";
	print LOG "Result: \"$result\"\n";
    }
    
    if($commit)
    {
	if(!$comment)
	{
	    $comment = "Added $elem";
	}
	
	commitToSVN($comment, undef, $user, $version_key); 
    }
}

# Commit any pending changes to subversion
#
sub commitToSVN
{
    my $comment     = shift; 
    my $path        = shift; 
    my $user        = shift; 
    my $version_key = shift; 

    my $command   = "$svn_command commit --non-interactive";
    my $result    = "";

    if($use_real_users && $user)
    {
	$command .= " --username " . qq{"$user"};
	$command .= " --password " . qq{"$user_password"};
    }

    $comment  = "Converstion from ClearCase" if not $comment; 
    $command .= qq{ -m"} . escape ($comment) . qq{"}; 

    
    $path = $svn_top_path if not $path; 
    $command = "cd \"$path\"; $command"; 

	
    if(!$test_mode)
    {
	$result = `$command 2>&1`;
    }
    else
    {
	$result = "Would have executed \"$command\"";
    }
    
    if($log_level > 2)
    {
	print LOG "Commit with command: \"$command\"\n";
	print LOG "Commit result: \"$result\"\n"; 
    }
    
    # there may be no result if the commit did not actually
    # occur which can happen when two versions are identical 
    # which, again, can happen if someone forced a commit to 
    # ClearCase despite no changes in the file (poop).
    #
    if(($result) && ($result !~ m/Committed revision/i))
    {
	if($commit_retries_left)
	{
	    my $msg = "commit failed: $result\n";
	    $msg .= "Will sleep for $retry_in_seconds seconds and try again\n";
	    $msg .= "$commit_retries_left times left to retry\n"; 
	    print $msg;

	    if($log_level > 0)
	    {
		print LOG $msg;
	    }

	    $commit_retries_left--; 
	    sleep($retry_in_seconds);
	    commitToSVN($comment, $path, $user, $version_key); 
	}
	else
	{
	    print "Commit failed, retry is either not set or we already tried, exiting\n";
	    print "$result\n";
	    doExit();
	}
    }
    else
    {

	$now = scalar(localtime); 
	$commit_retries_left = $retry_commits;; 

	if($version_key)
	{
	    $ver_hash{$version_key} = $now; 
	}
    }
    
    return $result; 
}



# Trims white space
#
sub trim
{
    my @out = @_;

    for (@out)
    {
	s/^\s+//;
	s/\s+$//;
    }
    return wantarray ? @out : $out[0];
}

# Escape quotes & such so comments don't break!
#
sub escape
{
    my @out = @_;
    
    for(@out)
    {
        s/[\x80-\xff]+/./g; 
	s/\"/\\\"/g;
    }
    return wantarray ? @out : $out[0];
}

# Escape spaces and other abominations in paths
#
sub path_escape
{
    my @out = @_;
    for (@out)
    {
	$_ = trim($_); 
	s/\s+/\\ /g;
	s/&+/_/g;
    }
    return wantarray ? @out : $out[0];
}

sub remainder_path 
{        
    my $fixedpath = shift;        
    my $fullpath  = shift;     

    # check for exact match (with or without trailing slashes)        
    #
    $fixedpath =~ s|/*$||;  # dump trailing slash        

    return ""       if $fullpath eq $fixedpath;        
    $fixedpath .= "/";        
    
    return ""       if $fullpath eq $fixedpath;        
    
    return $1       if $fullpath =~ m|^$fixedpath(.*)$|;        
    
    return "";      # no match
}

sub checkForExit
{
    if( -f "$working_dir/$exit_file")
    {
	my $msg = "$exit_file exists, script will exit\n"; 
	print LOG $msg if $log_level > 0;
	print $msg;

	doExit();
    }

    if( -f $sleep_file)
    {
	my $msg = "$sleep_file exists, script will sleep for 60s and try again\n"; 
	print $msg;
	print LOG $msg if $log_level > 0; 

	sleep(60);
	checkForExit(); 
    }
}

sub doExit
{
    open (USERSFILE, ">>$working_dir/$user_file");
    print USERSFILE "[users]\n"; 

    foreach my $user (keys %users_list)
    {
	print USERSFILE "$user = foo\n"; 
    }
    
    close (USERSFILE); 




    if($log_level > 0)
    {
	$now = scalar(localtime);
	print LOG "$now: Finished conversion\n";
	print LOG "Highest number of revisions per file was $rev_high_mark for $rev_high_file\n"; 
	print LOG "$num_directories directories and $num_files files converted\n"; 
	close (LOG);
    }
    exit 0;
}



__END__

=head1 NAME

convert_clearcase - convert ClearCase repository to SubVersion

=head1 SYNOPSIS

convert_clearcase [I<options>]

=head1 DESCRIPTION

This program converts a ClearCase repository to a SubVersion one.
For each object in ClearCase, it retrieves historical information,
gets the version data and checks each one into SubVersion. It
remembers the original checking-in user, date and command as
well as each revision comment.

Note: this was only tested on one specific ClearCase repository that
does not use branching. Use this at your own risk.

This program takes I<many> options (many of them required), and it's
not really practical to run this command multiple times by typing
them all on the command line. Instead, it's much better to make a
purpose-built script that calls it with all the options that you
need. Not only is this easier, but it insures that each run has
the same parameters.

   # run-conversion
   exec perl convert_clearcase.pl \
		--userealusers \
		--lcaseusers \
		--logfile="conversion_log" \
		--loglevel="3" \
		--ccpath="/home/src/" \
		--ccsubpath="com/mycompany" \
		--skipdir="ejbcgen" \
		--svnserver="192.26.158.1" \
		--svnrepos="my-repository" \
		--svnuser="svnuser" \
		--svnpass="foo"


=head1 NON-PERL PACKAGES REQUIRED

Berkeley DB files - use 4.2.x or higher as 4.1.x has some issues
which may cause SVN repository corruption, much slower too. 


=head1 OPTIONS

=over 4

=item B<--help>

Show a brief help listing to the standard error, then exit with success status.

=item B<--testmode>

Show the commands we would execute, but don't actually execute them; useful for
testing and finding the scope of a conversion.

=item B<--userealusers>

Only set this if the users in clear case history are existing SVN users
otherwise commits will fail. 

Easy way to work with this, run a test conversion first with this setting 
at 0, then use the above mentioned 'userfile' as the user file for Subversion
(in repository/conf/). Do a real run with this set to '1' and you'll guaranteed
the users exist! Of course, since all have same password you may want to remove
them right after the conversion and go back to your normal security model. 

=item B<--lcaseusers>

Convert usernames to lower case when checking files into subversion; 
see also the B<--userealusers> parameter.

=item B<--logfile=F>

Save log output to file I<F>; see also the B<--loglevel> parameter.

=item B<--loglevel=N>

Set the level of verbosity in the logfiles (specified with B<--logfile>) to I<N>.

0=none; 1=summary; 2=more; 3=lots

=item B<--cccomand=C>

The command used to invoke ClearCase. 

=item B<--ccpath=PATH>

Use I<PATH> as the source of the ClearCase tree to convert. This parth of the path 
is not pre-pended to the Subversion path; see also B<--ccsubpath>

This option is required. 

=item B<--ccsubpath=PATH>

Sub path to clear case tree, this will be appended to B<--ccpath>
Used if there's more than one set of source trees under the  default path and only one should be converted. 
This path is pre-pended to the SVN path, so "svn-repository/path/"  
Will become "svn-repository/this-path/path-from-conversion"; see also B<--ccpath>

=item B<--ccbranch=B>

Process only ClearCase branch I<B> into Subversion; only one branch
can be converted at a time. Default is "main".

=item B<--skipdir=D>

ClearCase Directories to skip when converting. - This is used in a regex so separate by | 
This is only used when generating intial subversion paths, it is 
assumed that there are no ClearCase files in these directories. 

=item B<--svncommand=C>

Command to invoke subversion defaults to "svn"

=item B<--svnserver=SVR>

Use I<SVR> - hostname or IP - for the SubVersion server. Defaults to localhost.

=item B<--svnctype=TYPE>

Use connection type I<TYPE> when connection to the SubVersion server.
Valid options are B<svn>, B<svn+ssh>, B<file> and B<http>, but we've
only tested B<svn> (which is the default).

=item B<--svnrepos=REPOS>

Specify the SubVersion repository as I<REPOS>. This option is required.

=item B<--svnrepospath=PATH>

Path to subversion repository. Default is /home/B<--svnrepos>.

=item B<--svnuser=USER>

Use I<USER> as the Subversion user.

=item B<--svncpass=PASS>

Use I<PASS> as the password for the SubVersion repository.

=item B<--workingdir=DIR>

Use I<DIR> as the working directory for the script. Defaults to current working directory.

=item B<--datelimit=DD-MM-YYYY>

Process only history going back to this date, in DD-MM-YYYY form.
If B<--countlimit> is also specified, the first limit reached is
dispositive.

=item B<--countlimit=N>

Process only I<N> events, exiting thereafter. If B<--datelimit>
is also specified, the first limit reached is dispositive.

=item B<--userfile=F>

Provide a filename to dump all the user data into; see also B<--userealusers>

=item B<--commits=N>

If commit fails, sleep and retry up to I<N> times. Defaults to 10.

=item B<--commitsleep=N>

If commit fails, sleep for I<N> seconds before retrying. Defaults to 60.

=item B<--exitfile=F>

If this file is found in the working directory, this program
exits gracefully after saving state and cleaning up properly.
One need only touch the file to trigger this.

=item B<--dbfile=F>

File in which to store conversion history information. This is used if running the conversion 
more than once. Each time a version of a file is converted, the script stores that information
and will not attempt to convert the same version again -- only any non-converted ones. 

=item B<--sleepfile=F>
 
While this file exists the script will sleep and resume its processing when the file
is removed.  See also B<--exitfile>

=item B<--nocheckout>

Use this option if you'd like to skip the subversion checkout step. Although, if your copy
of the svn repository is not up to date the conversion will fail. 

=back

=head1 BUGS/MISFEATURES

This program is not a comprehensive conversion tool and has some limits.

It doesn't deal with branching - only one branch is supported

It doesn't convert directory information; it generates it
by checking in each version.

It doesn't retrieve information for deleted files; only files that
are currently visible in ClearCase are converted.

It doesn't retrieve information on moved files, only the existing version.

ClearCase supports spaces in user names; SubVersion does not. Spaces
in user names are removed during conversion.

=head1 AUTHOR

Kasia Trapszo C<ktrapszo@unix-girl.com>

=cut

doc.html

NAME
convert_clearcase - convert ClearCase repository to SubVersion

SYNOPSIS
convert_clearcase [options]

DESCRIPTION
This program converts a ClearCase repository to a SubVersion one. For each object in ClearCase, it retrieves historical information, gets the version data and checks each one into SubVersion. It remembers the original checking-in user, date and command as well as each revision comment.

Note: this was only tested on one specific ClearCase repository that does not use branching. Use this at your own risk.

This program takes many options (many of them required), and it's not really practical to run this command multiple times by typing them all on the command line. Instead, it's much better to make a purpose-built script that calls it with all the options that you need. Not only is this easier, but it insures that each run has the same parameters.

   # run-conversion
   exec perl convert_clearcase.pl \
                --userealusers \
                --lcaseusers \
                --logfile="conversion_log" \
                --loglevel="3" \
                --ccpath="/home/src/" \
                --ccsubpath="com/mycompany" \
                --skipdir="ejbcgen" \
                --svnserver="192.26.158.1" \
                --svnrepos="my-repository" \
                --svnuser="svnuser" \
                --svnpass="foo"
NON-PERL PACKAGES REQUIRED
Berkeley DB files - use 4.2.x or higher as 4.1.x has some issues which may cause SVN repository corruption, much slower too.

OPTIONS
--help
Show a brief help listing to the standard error, then exit with success status.
--testmode
Show the commands we would execute, but don't actually execute them; useful for testing and finding the scope of a conversion.
--userealusers
Only set this if the users in clear case history are existing SVN users otherwise commits will fail.
Easy way to work with this, run a test conversion first with this setting at 0, then use the above mentioned 'userfile' as the user file for Subversion (in repository/conf/). Do a real run with this set to '1' and you'll guaranteed the users exist! Of course, since all have same password you may want to remove them right after the conversion and go back to your normal security model.

--lcaseusers
Convert usernames to lower case when checking files into subversion; see also the --userealusers parameter.
--logfile=F
Save log output to file F; see also the --loglevel parameter.
--loglevel=N
Set the level of verbosity in the logfiles (specified with --logfile) to N.
0=none; 1=summary; 2=more; 3=lots

--cccomand=C
The command used to invoke ClearCase.
--ccpath=PATH
Use PATH as the source of the ClearCase tree to convert. This parth of the path is not pre-pended to the Subversion path; see also --ccsubpath
This option is required.

--ccsubpath=PATH
Sub path to clear case tree, this will be appended to --ccpath Used if there's more than one set of source trees under the default path and only one should be converted. This path is pre-pended to the SVN path, so ``svn-repository/path/'' Will become ``svn-repository/this-path/path-from-conversion''; see also --ccpath
--ccbranch=B
Process only ClearCase branch B into Subversion; only one branch can be converted at a time. Default is ``main''.
--skipdir=D
ClearCase Directories to skip when converting. - This is used in a regex so separate by | This is only used when generating intial subversion paths, it is assumed that there are no ClearCase files in these directories.
--svncommand=C
Command to invoke subversion defaults to ``svn''
--svnserver=SVR
Use SVR - hostname or IP - for the SubVersion server. Defaults to localhost.
--svnctype=TYPE
Use connection type TYPE when connection to the SubVersion server. Valid options are svn, svn+ssh, file and http, but we've only tested svn (which is the default).
--svnrepos=REPOS
Specify the SubVersion repository as REPOS. This option is required.
--svnrepospath=PATH
Path to subversion repository. Default is /home/--svnrepos.
--svnuser=USER
Use USER as the Subversion user.
--svncpass=PASS
Use PASS as the password for the SubVersion repository.
--workingdir=DIR
Use DIR as the working directory for the script. Defaults to current working directory.
--datelimit=DD-MM-YYYY
Process only history going back to this date, in DD-MM-YYYY form. If --countlimit is also specified, the first limit reached is dispositive.
--countlimit=N
Process only N events, exiting thereafter. If --datelimit is also specified, the first limit reached is dispositive.
--userfile=F
Provide a filename to dump all the user data into; see also --userealusers
--commits=N
If commit fails, sleep and retry up to N times. Defaults to 10.
--commitsleep=N
If commit fails, sleep for N seconds before retrying. Defaults to 60.
--exitfile=F
If this file is found in the working directory, this program exits gracefully after saving state and cleaning up properly. One need only touch the file to trigger this.
--dbfile=F
File in which to store conversion history information. This is used if running the conversion more than once. Each time a version of a file is converted, the script stores that information and will not attempt to convert the same version again -- only any non-converted ones.
--sleepfile=F

While this file exists the script will sleep and resume its processing when the file
is removed.  See also B<--exitfile>
BUGS/MISFEATURES
This program is not a comprehensive conversion tool and has some limits.

It doesn't deal with branching - only one branch is supported

It doesn't convert directory information; it generates it by checking in each version.

It doesn't retrieve information for deleted files; only files that are currently visible in ClearCase are converted.

It doesn't retrieve information on moved files, only the existing version.

ClearCase supports spaces in user names; SubVersion does not. Spaces in user names are removed during conversion.

AUTHOR
Kasia Trapszo ktrapszo@unix-girl.com

example.sh

exec perl convert_clearcase.pl \
		--userealusers \
		--lcaseusers \
		--logfile="conversion_log" \
		--loglevel="3" \
		--ccpath="/home/vobadm/src/" \
		--ccsubpath="/com/mycompany/" \
		--skipdir="ejbcgen" \
		--svnserver="10.1.1.3" \
		--svnrepos="my-repository" \
		--svnuser="svnuser" \
		--svnpass="foo"