Friday, October 31, 2008

kopier, a multi-threaded backup utility in C#

Kopier manages a pool of threads, each thread managing an instance of Robocopy. Some logging and error handling is built in to her as well.

the program class:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Net.Mail;

namespace kopier
{
    class Program
    {
        const int maxThreads = 8;

        static void Main(string[] args)
        {
            DateTime startTime;
            int w, c;
            StringBuilder aggMsg = new StringBuilder();

            string[] lines = System.IO.File.ReadAllLines(@args[0]);

            ManualResetEvent[] doneEvents = new ManualResetEvent[lines.Count()];
            kopy[] kopArray = new kopy[lines.Count()];

            ThreadPool.SetMaxThreads(maxThreads, 100);
            ThreadPool.GetMaxThreads(out w, out c);

            Console.WriteLine("--------------------------------------------------------------------------------");
            Console.WriteLine("kopier");
            Console.WriteLine("--------------------------------------------------------------------------------");
            Console.WriteLine("Max threads: " + w + ", " + c);
            Console.WriteLine("Launching {0} jobs...", lines.Count());
            startTime = DateTime.Now;
           
            for (int i = 0; i < lines.Count(); i++)
            {
                doneEvents[i] = new ManualResetEvent(false);
                string[] switches = lines[i].Split(new char[]{','});
                string src = switches[0];
                string dst = switches[1];
                TimeSpan min = TimeSpan.FromMinutes(double.Parse(switches[2]));
                kopy k = new kopy(src, dst, min, doneEvents[i]);
                kopArray[i] = k;
                ThreadPool.QueueUserWorkItem(k.threadPoolCallback, i);
            }

            WaitHandle.WaitAll(doneEvents);
            Console.WriteLine("All jobs are complete.");

            aggMsg.Append("started: " + startTime.ToShortTimeString() + ", " + startTime.ToShortDateString() + "\n");
            aggMsg.Append("finished: " + DateTime.Now.ToShortTimeString() + ", " + DateTime.Now.ToShortDateString() + "\n");
            aggMsg.Append("duration: " + DateTime.Now.Subtract(startTime).ToString() + "\r \n");

            for (int i = 0; i < lines.Count(); i++)
            {
                kopy k = kopArray[i];
                Console.WriteLine("job({0}): {1}", k.srcDir, k.kopyMsg);
                aggMsg.Append("job(" + k.srcDir + "): " + k.kopyMsg + "\r \n");
            }

            sendMsg(aggMsg.ToString());
        }

        private static void sendMsg(string msg)
        {
            try
            {
                MailMessage email = new MailMessage();
                email.From = new MailAddress("results@kopy.int");
                email.To.Add(new MailAddress("elijahb@bargain.com"));
                email.Subject = "kopy results";
                email.Body = msg;
                SmtpClient client = new SmtpClient("172.16.12.25");
                client.Send(email);
            }
            catch (Exception e)
            {
                Console.WriteLine("Exception during email: {0}", e);
            }
        }
    }
}


the kopy class:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using System.Threading;
using System.Diagnostics;

namespace kopier
{
    class kopy
    {
        public kopy(string srcDir, string dstDir, TimeSpan timeMin, ManualResetEvent doneEvent)
        {
            _srcDir = srcDir;
            _dstDir = dstDir;
            _timeMin = timeMin;
            _doneEvent = doneEvent;
        }

        public void threadPoolCallback(object threadContext)
        {
            int threadIndex = (int)threadContext;
            _startTime = DateTime.Now;
 
            Console.Write("job {0} started..." +
                "\n \t job {0} source = " + _srcDir +
                "\n \t job {0} destination = " + _dstDir +
                "\n \t job {0} time limit = " + _timeMin.TotalMinutes.ToString() + " minutes" +
                "\n", threadIndex);
 
            _kopyMsg = kopyAway(_srcDir, _dstDir, _timeMin);

            Console.Write("job {0} finished..." +
                "\n \t job {0} source = " + _srcDir +
                "\n \t job {0} destination = " + _dstDir +
                "\n \t job {0} duration = " + DateTime.Now.Subtract(_startTime).ToString() +
                "\n", threadIndex);

            _doneEvent.Set();
        }

        private string kopyAway(string src, string dst, TimeSpan min)
        {
            string log = dst + @"\rcLog(" + DateTime.Today.ToShortDateString().Replace('/', '-') + ").txt";
            string msg;
            Process p = new Process();

            if (System.IO.Directory.Exists(dst) != true)
            {
                System.IO.Directory.CreateDirectory(dst);
            }

            dst = dst + @"\files";

            try
            {
                p.StartInfo.FileName = "ROBOCOPY";
                p.StartInfo.Arguments = '"' + src + '"' + " " + '"' + dst + '"' + " /MIR /W:10 /R:3 /NP /TEE /LOG+:" + '"' + log + '"';
                p.Start();

                do
                {
                    if (DateTime.Now.Subtract(p.StartTime) >= min)
                    {
                        p.Kill();
                        throw new Exception(buildOutput(src, dst, p.StartTime, DateTime.Now, "Failed: killed for exceeding max time."));
                    }
                    System.Threading.Thread.Sleep(500);
                } while (p.HasExited == false);

                msg = buildOutput(src, dst, p.StartTime, DateTime.Now, "Succeeded.");
            }
            catch (Exception e)
            {
                msg = e.Message.ToString();
            }
            finally
            {
                p.Dispose();
            }
            return msg;
        }

        private string buildOutput(string src, string dst, DateTime startTime, DateTime endTime, string result)
        {  
            return result +
                "\n \t source: " + src +
                "\n \t destination: " + dst +
                "\n \t started: " + startTime.ToShortTimeString() + ", " + startTime.ToShortDateString() +
                "\n \t finished: " + endTime.ToShortTimeString() + ", " + endTime.ToShortDateString() +
                "\n \t duration: " + endTime.Subtract(startTime).ToString();
        }

        public string srcDir { get { return _srcDir; } }
        private string _srcDir;

        public string dstDir { get { return _dstDir; } }
        private string _dstDir;

        public TimeSpan timeMin { get { return _timeMin; } }
        private TimeSpan _timeMin;

        public string kopyMsg { get { return _kopyMsg; } }
        private string _kopyMsg;

        private ManualResetEvent _doneEvent;

        private DateTime _startTime;
    }
}

No comments: