[devel] MySync

[devel] MySync

[Kirill A. Shutemov]

[-- Attachment #1.1: Type: text/plain, Size: 1464 bytes --]

Тут для себя написал скриптик. Может кому будет интересно...

Что-то типа оффлайного rsync'а -- позволяет побновлять одно дерево файлов
до другого. Предпологается возможность перетаскивать обновления Сизифа 
(скажем, с работы домой), пемерно следующей последовательностью команд:

Дома: ./mysync -s каталог_с_сизифом -o Sisyphus.shot
	Это довольно долгий процесс - у меня на машине(Cel-300A, 320Mb)
	~20 минут. Далее отсылаем полученый файл(в bzip'е ~400kb) на
	работу

На работе: ./mysync -c каталог_с_сизифом Sisyphus.shot -o Sisyphus.tar
	Залеваем Sisypus.tar на компакт или иной носитель, несем домой

Дома: ./mysync -u -r  Sisyphus.tar каталог_с_сизифом
	Вуаля... :)

За подробностями смотрите --help.

Если кого-нить эта поделка заинтересовала могу положить в rpm.

P.S. Требует наличие в системе ruby, tar, diff и md5sum. 

P.P.S. Замена последнего на рубёвый(см. закоменченый код) вызвала увеличение 
времени выполнения первой команды в 2 раза и при этом запросто съедалась вся 
ОП + кусок свопа. Может кто объяснит почему это проиходит?

-- 
Kirill A. Shutemov
E-mail: kirya85@mail.ru
JID: kas@altlinux.org
ICQ: 152302675

Предыдущее (-alt3) и эти изменения исправляют ошибки, которые, насколько я
понимаю, благодаря технологии PrivilegeSeparation в худшем случае могут
привести к возможности исполнения кода с правами псевдопользователя в
пустом чруте.  К тому же что ещё и не известно, могут ли.
		-- ldv in devel@

[-- Attachment #1.2: mysync --]
[-- Type: text/plain, Size: 8266 bytes --]

#!/usr/bin/env ruby

require 'find'
require 'ftools'
#require 'md5'
require 'tempfile'


$verbose = false
$remove = false
$exit_on_warning = false
$exit_on_error = true

def version 
    "0.1.0"
end

def fatal(msg)
    STDERR.puts("FATAL!: " + msg)
    exit(2)
end

def error(msg)
    STDERR.puts("ERROR!: " + msg)
    exit(1) if $exit_on_error
end

def warning(msg)
    STDERR.puts("WARNING!: " + msg)
    exit(1) if $exit_on_warning
end

def snapshot(path, output=STDOUT)
    STDERR.puts "Generation snapshot for #{path}..." if $verbose	
    
    fatal("#{path} isn't directory") unless FileTest.directory?(path)
    
    output.puts("# MySync snapshot version #{version}", "# Date: #{Time.now}")
    
    arr = []
    Find.find(path){|x|
	arr.push(x)
    }
    arr.sort!
    arr.shift
    
    arr.each{|s|
	x = s.sub(/^#{path}\/?/, '')
	case File.lstat(s).ftype	    
	when "directory"
	    STDERR.puts x if $verbose	
	    output.puts("D #{x}")
	when "file"
#	    f = File.open(s)
#	    output.puts("F #{x} #{MD5.md5(f.readlines.to_s)}")
#	    f.close
	    output.puts("F #{x} " + `md5sum #{s}`.sub(/ [^ ]+$/,''))
	when "link"
	    output.puts("L #{x} #{File.readlink(s)}")
	else
	    warning("Not supported file type: #{s} - #{File.lstat(s).ftype}.")
	end
	output.flush
    }
    STDERR.puts "Generation snapshot for #{path} done." if $verbose	
    
    return output
end

def diff(to, from, output=STDOUT)
    STDERR.puts "Generation diff between #{from} and #{to}..." if $verbose	
    
    fatal("#{from} - no such file or directory.") unless FileTest.exist?(from)
    
    fatal("#{to} - no such file or directory.") unless FileTest.exist?(to)
    
    output.puts("# MySync Diff version #{version}", "# Date: #{Time.now}")
    
    if File.open(from).stat.directory?
	tf = Tempfile.new("from")
	snapshot(from, tf)
	from = tf.path
    end
    
    if File.open(to).stat.directory?
	tf = Tempfile.new("to")
	snapshot(to, tf)
	to = tf.path
    end
    
    arr = `diff -U 0 #{from} #{to}`.split(/\n/)
    arr.delete_if{|s| not s =~ /^[+-][FDL]/}
    
    new = []
    remove = []
    change = []
    
    arr.each{|s|
	path = s.gsub(/[+-][FDL] ([^ ]+).*/, '\1')
	
	if s=~/^\+/
	    new.push(path)
	else
	    remove.push(s.gsub(/[+-][FDL] ([^ ]+).*/, '\1'))
	end
    }
    
    remove.each{|s| change.push(s) if new.include?(s)}
    change.each{|s|
	new.delete(s)
	remove.delete(s)
    }
    change.each{|s| output.puts("C #{s}")}
    new.each{|s| output.puts("N #{s}")}
    remove.each{|s| output.puts("R #{s}")}
    
    output.flush	
    STDERR.puts "Generation diff between #{from} and #{to} done" if $verbose	
    
    return output
end

def create(to, from, output=STDOUT)
    STDERR.puts "Generation tar with changes between #{from} and #{to}" if $verbose	
    
    fatal("#{from} - no such file or directory.") unless FileTest.exist?(from)
    
    fatal("#{to} isn't directory.") unless FileTest.directory?(to)
    
    
    to = File.expand_path(to)
    
    tf = Tempfile.new(".mysync")
    
    diff(to, from, tf)
    
    arr = File.open(tf.path).readlines	
    
    tar = ""
    arr.each{|s|
	tar += " " + s.sub(/^[NC] (.*)\n$/, '\1') if s =~ /^[NC] /
    }
    
    STDERR.puts "Packing tar #{from}" if $verbose	
    output.puts(`tar --no-recursion -c -C #{tf.path.gsub(/\/([^\/]+$)/,' \1')} -C #{to} #{tar} \
#{ $verbose ? '--verbose' : ''}`)
    output.flush
    
    STDERR.puts "Generation tar with changes between #{from} and #{to} done" if $verbose	
    
    return output
end

def update(from, to)
    STDERR.puts "Updating #{to} from #{from}" if $verbose	
    
    fatal("#{to} isn't directory.") unless FileTest.directory?(to)
    
    from = File.expand_path(from)
    
    if FileTest.directory?(from)
	tf = Tempfile.new("tar")
	create(from, to, tf)
	from = tf.path
    end
    
    arr = `tar -O -xf #{from} ".mysync*"`.split("\n")

    pwd = Dir.pwd
    Dir.chdir(to)
    
    remove = []
    
    arr.each{|s|
	a = /^([NCR]) (.*)$/.match(s).to_a
	
	case a[1]
	when "N"
	    error("File #{a[2]} labeled as new, but exist in the dir.") if
		FileTest.exist?(a[2])
	when "C"
	    warning("File #{a[2]} labeled as changed, but not exist in the dir.") if
		!FileTest.exist?(a[2])
	when "R"
	    warning("File #{a[2]} labeled as removable, but not exist in the dir.") if
		!FileTest.exist?(a[2])
	    remove.push(a[2])
	end
    }
    
    STDERR.puts "Unpacking tar #{from}" if $verbose	
    
    error("Untaring files failure.") unless 
	system("tar -xf #{from} #{ $verbose ? '--verbose' : ''}")
    
    remove.reverse_each{|file| File.rm_f(file) or Dir.rmdir(file)} if $remove
    Dir[".mysync*"].each{|file| File.rm_f(file)}
    Dir.chdir(pwd)
    
    STDERR.puts "Updating #{to} from #{from} done" if $verbose	
end

if __FILE__ == $0
    require 'getoptlong'

    def show_usage
	puts "Usage: mysync -s [-o file] [-v] DIR"
	puts "  or   mysync -d [-o file] [-v] (NEWDIR|SNAPSHOT) (ORIGDIR|SNAPSHOT)"
	puts "  or   mysync -c [-o file] [-v] (NEWDIR) (ORIGDIR|SNAPSHOT)"
	puts "  or   mysync -u [-v] [-r] (SRCDIR|TAR) DSTDIR"
	puts "  or   mysync -h"
	puts
	puts "Options"
	puts " -s, --snapshot\t\tCreate snapshot of DIR"
	puts " -d, --diff\t\tCompare two directories or snapshots"
	puts " -c, --create\t\tCreate tar with changes between ORIGDIR or SNAPSHOT and NEWDIR)"
	puts " -u, --update\t\tUpdate DSTDIR to SRCDIR or TAR(created by --create)"
	puts " -r, --remove\t\tRemove files in DSTDIR wich didn't exist in SRCDIR(only with -u)"
	puts " -o, --output=FILE\tWrite output into FILE (not with --update)"
	puts " -v, --verbose\t\tBe verbose(very verbose %), print into stderr"
	puts " -w, --warning-exit\tExit on a WARNING"
	puts " -e, --error-continue\tDon't exit on ERROR"
	puts " -h, --help\t\tShow this usage"
	exit
    end

    opts = GetoptLong.new(
			  [ "--snapshot", "-s", GetoptLong::NO_ARGUMENT],
			  [ "--diff", "-d", GetoptLong::NO_ARGUMENT],
			  [ "--create", "-c", GetoptLong::NO_ARGUMENT],
			  [ "--update", "-u", GetoptLong::NO_ARGUMENT],
			  [ "--remove", "-r", GetoptLong::NO_ARGUMENT],
			  [ "--output", "-o", GetoptLong::REQUIRED_ARGUMENT],
			  [ "--verbose", "-v", GetoptLong::NO_ARGUMENT],
			  [ "--warning-exit", "-w", GetoptLong::NO_ARGUMENT],
			  [ "--error-continue", "-e", GetoptLong::NO_ARGUMENT],
			  [ "--help", "-h", GetoptLong::NO_ARGUMENT]
			  )
    mode = filename = ""

    opts.each{|opt,arg|
	case opt
	when "--help"
	    show_usage
	when "--snapshot"
	    fatal("You only can select one mode.") unless mode == ""
	    mode = "snapshot"
	when "--diff"
	    fatal("You only can select one mode.") unless mode == ""
	    mode = "diff"
	when "--create"
	    fatal("You only can select one mode.") unless mode == ""
	    mode = "create"
	when "--update"
	    fatal("You only can select one mode.") unless mode == ""
	    mode = "update"
	when "--remove"
	    $remove = true
	when "--output"
	    filename = arg
	when "--verbose"
	    $verbose = true
	when "--warning-exit"
	    $exit_on_warning = true
	when "--error-continue"
	    $exit_on_error = false
	end
    }

    error("--output can't be used with --update") if mode == "update" and filename != ""
    error("--remove only can use with --update") if $remove and mode != "update"

    case mode
    when "snapshot"
	show_usage unless ARGV.size == 1
	if filename != ""
	    snapshot(ARGV[0], File.new(filename, File::CREAT|File::TRUNC|File::RDWR))
	else
	    snapshot(ARGV[0])
	end
	exit
    when "diff"
	show_usage unless ARGV.size == 2
	if filename != ""
	    diff(ARGV[0], ARGV[1],
			    File.new(filename, File::CREAT|File::TRUNC|File::RDWR))
	else
	    diff(ARGV[0], ARGV[1])
	end
	exit
    when "create"
	show_usage unless ARGV.size == 2
	if filename != ""
	    create(ARGV[0], ARGV[1],
			    File.new(filename, File::CREAT|File::TRUNC|File::RDWR))
	else
	    create(ARGV[0], ARGV[1])
	end
	exit
	
    when "update"
	show_usage unless ARGV.size == 2
	update(ARGV[0], ARGV[1])
	exit
    else
	show_usage
    end
end

[-- Attachment #2: Type: application/pgp-signature, Size: 189 bytes --]