small typo in debug message
[svn2git:marcguenther-svn2git.git] / src / repository.cpp
1 /*
2  *  Copyright (C) 2007  Thiago Macieira <thiago@kde.org>
3  *  Copyright (C) 2009 Thomas Zander <zander@kde.org>
4  *
5  *  This program is free software: you can redistribute it and/or modify
6  *  it under the terms of the GNU General Public License as published by
7  *  the Free Software Foundation, either version 3 of the License, or
8  *  (at your option) any later version.
9  *
10  *  This program is distributed in the hope that it will be useful,
11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  *  GNU General Public License for more details.
14  *
15  *  You should have received a copy of the GNU General Public License
16  *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
17  */
18
19 #include "repository.h"
20 #include "CommandLineParser.h"
21 #include <QTextStream>
22 #include <QDebug>
23 #include <QDir>
24 #include <QLinkedList>
25
26 static const int maxSimultaneousProcesses = 100;
27
28 class ProcessCache: QLinkedList<Repository *>
29 {
30 public:
31     void touch(Repository *repo)
32     {
33         remove(repo);
34
35         // if the cache is too big, remove from the front
36         while (size() >= maxSimultaneousProcesses)
37             takeFirst()->closeFastImport();
38
39         // append to the end
40         append(repo);
41     }
42
43     inline void remove(Repository *repo)
44     {
45 #if QT_VERSION >= 0x040400
46         removeOne(repo);
47 #else
48         removeAll(repo);
49 #endif
50     }
51 };
52 static ProcessCache processCache;
53
54 Repository::Repository(const Rules::Repository &rule)
55     : name(rule.name), commitCount(0), outstandingTransactions(0), processHasStarted(false)
56 {
57     foreach (Rules::Repository::Branch branchRule, rule.branches) {
58         Branch branch;
59         branch.created = 0;     // not created
60
61         branches.insert(branchRule.name, branch);
62     }
63
64     // create the default branch
65     branches["master"].created = 1;
66
67     fastImport.setWorkingDirectory(name);
68     if (!CommandLineParser::instance()->contains("dry-run")) {
69         if (!QDir(name).exists()) { // repo doesn't exist yet.
70             qDebug() << "Creating new repository" << name;
71             QDir::current().mkpath(name);
72             QProcess init;
73             init.setWorkingDirectory(name);
74             init.start("git", QStringList() << "--bare" << "init");
75             init.waitForFinished(-1);
76         }
77     }
78 }
79
80 Repository::~Repository()
81 {
82     Q_ASSERT(outstandingTransactions == 0);
83     closeFastImport();
84 }
85
86 void Repository::closeFastImport()
87 {
88     if (fastImport.state() != QProcess::NotRunning) {
89         fastImport.write("checkpoint\n");
90         fastImport.waitForBytesWritten(-1);
91         fastImport.closeWriteChannel();
92         if (!fastImport.waitForFinished()) {
93             fastImport.terminate();
94             if (!fastImport.waitForFinished(200))
95                 qWarning() << "git-fast-import for repository" << name << "did not die";
96         }
97     }
98     processHasStarted = false;
99     processCache.remove(this);
100 }
101
102 void Repository::reloadBranches()
103 {
104     QProcess revParse;
105     revParse.setWorkingDirectory(name);
106     revParse.start("git", QStringList() << "rev-parse" << "--symbolic" << "--branches");
107     revParse.waitForFinished(-1);
108
109     if (revParse.exitCode() == 0 && revParse.bytesAvailable()) {
110         while (revParse.canReadLine()) {
111             QByteArray branchName = revParse.readLine().trimmed();
112
113             //qDebug() << "Repo" << name << "reloaded branch" << branchName;
114             branches[branchName].created = 1;
115             fastImport.write("reset refs/heads/" + branchName +
116                              "\nfrom refs/heads/" + branchName + "^0\n\n"
117                              "progress Branch refs/heads/" + branchName + " reloaded\n");
118         }
119     }
120 }
121
122 void Repository::createBranch(const QString &branch, int revnum,
123                               const QString &branchFrom, int)
124 {
125     startFastImport();
126     if (!branches.contains(branch)) {
127         qWarning() << branch << "is not a known branch in repository" << name << endl
128                    << "Going to create it automatically";
129     }
130
131     QByteArray branchRef = branch.toUtf8();
132     if (!branchRef.startsWith("refs/"))
133         branchRef.prepend("refs/heads/");
134
135     Branch &br = branches[branch];
136     if (br.created && br.created != revnum) {
137         QByteArray backupBranch = branchRef + '_' + QByteArray::number(revnum);
138         qWarning() << branch << "already exists; backing up to" << backupBranch;
139
140         fastImport.write("reset " + backupBranch + "\nfrom " + branchRef + "\n\n");
141     }
142
143     // now create the branch
144     br.created = revnum;
145     QByteArray branchFromRef = branchFrom.toUtf8();
146     if (!branchFromRef.startsWith("refs/"))
147         branchFromRef.prepend("refs/heads/");
148
149     if (!branches.contains(branchFrom) || !branches.value(branchFrom).created) {
150         qCritical() << branch << "in repository" << name
151                     << "is branching from branch" << branchFrom
152                     << "but the latter doesn't exist. Can't continue.";
153         exit(1);
154     }
155
156     fastImport.write("reset " + branchRef + "\nfrom " + branchFromRef + "\n\n"
157                      "progress Branch " + branchRef + " created from "
158                      + branchFromRef + " r" + QByteArray::number(revnum) + "\n\n");
159 }
160
161 Repository::Transaction *Repository::newTransaction(const QString &branch, const QString &svnprefix,
162                                                     int revnum)
163 {
164     startFastImport();
165     if (!branches.contains(branch)) {
166         qWarning() << branch << "is not a known branch in repository" << name << endl
167                    << "Going to create it automatically";
168     }
169
170     Transaction *txn = new Transaction;
171     txn->repository = this;
172     txn->branch = branch.toUtf8();
173     txn->svnprefix = svnprefix.toUtf8();
174     txn->datetime = 0;
175     txn->revnum = revnum;
176
177     if ((++commitCount % CommandLineParser::instance()->optionArgument(QLatin1String("commit-interval"), QLatin1String("10000")).toInt()) == 0)
178         // write everything to disk every 10000 commits
179         fastImport.write("checkpoint\n");
180     if (outstandingTransactions++ == 0)
181         lastmark = 1;           // reset the mark number
182     return txn;
183 }
184
185 void Repository::createAnnotatedTag(const QString &ref, const QString &svnprefix,
186                                     int revnum,
187                                     const QByteArray &author, uint dt,
188                                     const QByteArray &log)
189 {
190     QString tagName = ref;
191     if (tagName.startsWith("refs/tags/"))
192         tagName.remove(0, 10);
193
194     if (!annotatedTags.contains(tagName))
195         printf("Creating annotated tag %s (%s)\n", qPrintable(tagName), qPrintable(ref));
196     else
197         printf("Re-creating annotated tag %s\n", qPrintable(tagName));
198
199     AnnotatedTag &tag = annotatedTags[tagName];
200     tag.supportingRef = ref;
201     tag.svnprefix = svnprefix.toUtf8();
202     tag.revnum = revnum;
203     tag.author = author;
204     tag.log = log;
205     tag.dt = dt;
206 }
207
208 void Repository::finalizeTags()
209 {
210     if (annotatedTags.isEmpty())
211         return;
212
213     printf("Finalising tags for %s...", qPrintable(name));
214     startFastImport();
215
216     QHash<QString, AnnotatedTag>::ConstIterator it = annotatedTags.constBegin();
217     for ( ; it != annotatedTags.constEnd(); ++it) {
218         const QString &tagName = it.key();
219         const AnnotatedTag &tag = it.value();
220
221         QByteArray message = tag.log;
222         if (!message.endsWith('\n'))
223             message += '\n';
224         if (CommandLineParser::instance()->contains("add-metadata"))
225             message += "\nsvn path=" + tag.svnprefix + "; revision=" + QByteArray::number(tag.revnum) + "\n";
226
227         {
228             QByteArray branchRef = tag.supportingRef.toUtf8();
229             if (!branchRef.startsWith("refs/"))
230                 branchRef.prepend("refs/heads/");
231
232             QTextStream s(&fastImport);
233             s << "progress Creating annotated tag " << tagName << " from ref " << branchRef << endl
234               << "tag " << tagName << endl
235               << "from " << branchRef << endl
236               << "tagger " << QString::fromUtf8(tag.author) << ' ' << tag.dt << " -0000" << endl
237               << "data " << message.length() << endl;
238         }
239
240         fastImport.write(message);
241         fastImport.putChar('\n');
242         if (!fastImport.waitForBytesWritten(-1))
243             qFatal("Failed to write to process: %s", qPrintable(fastImport.errorString()));
244
245         printf(" %s", qPrintable(tagName));
246         fflush(stdout);
247     }
248
249     while (fastImport.bytesToWrite())
250         if (!fastImport.waitForBytesWritten(-1))
251             qFatal("Failed to write to process: %s", qPrintable(fastImport.errorString()));
252     printf("\n");
253 }
254
255 void Repository::startFastImport()
256 {
257     if (fastImport.state() == QProcess::NotRunning) {
258         if (processHasStarted)
259             qFatal("git-fast-import has been started once and crashed?");
260         processHasStarted = true;
261
262         // start the process
263         QString outputFile = name;
264         outputFile.replace('/', '_');
265         outputFile.prepend("log-");
266         fastImport.setStandardOutputFile(outputFile, QIODevice::Append);
267         fastImport.setProcessChannelMode(QProcess::MergedChannels);
268
269         if (!CommandLineParser::instance()->contains("dry-run")) {
270             fastImport.start("git", QStringList() << "fast-import");
271         } else {
272             fastImport.start("/bin/cat", QStringList());
273         }
274
275         reloadBranches();
276     }
277 }
278
279 Repository::Transaction::~Transaction()
280 {
281     --repository->outstandingTransactions;
282 }
283
284 void Repository::Transaction::setAuthor(const QByteArray &a)
285 {
286     author = a;
287 }
288
289 void Repository::Transaction::setDateTime(uint dt)
290 {
291     datetime = dt;
292 }
293
294 void Repository::Transaction::setLog(const QByteArray &l)
295 {
296     log = l;
297 }
298
299 void Repository::Transaction::deleteFile(const QString &path)
300 {
301     QString pathNoSlash = path;
302     if(pathNoSlash.endsWith('/'))
303         pathNoSlash.chop(1);
304     deletedFiles.append(pathNoSlash);
305 }
306
307 QIODevice *Repository::Transaction::addFile(const QString &path, int mode, qint64 length)
308 {
309     int mark = ++repository->lastmark;
310
311     if (modifiedFiles.capacity() == 0)
312         modifiedFiles.reserve(2048);
313     modifiedFiles.append("M ");
314     modifiedFiles.append(QByteArray::number(mode, 8));
315     modifiedFiles.append(" :");
316     modifiedFiles.append(QByteArray::number(mark));
317     modifiedFiles.append(' ');
318     modifiedFiles.append(path.toUtf8());
319     modifiedFiles.append("\n");
320
321     if (!CommandLineParser::instance()->contains("dry-run")) {
322         repository->fastImport.write("blob\nmark :");
323         repository->fastImport.write(QByteArray::number(mark));
324         repository->fastImport.write("\ndata ");
325         repository->fastImport.write(QByteArray::number(length));
326         repository->fastImport.write("\n", 1);
327     }
328
329     return &repository->fastImport;
330 }
331
332 void Repository::Transaction::commit()
333 {
334     processCache.touch(repository);
335
336     // create the commit message
337     QByteArray message = log;
338     if (!message.endsWith('\n'))
339         message += '\n';
340     if (CommandLineParser::instance()->contains("add-metadata"))
341         message += "\nsvn path=" + svnprefix + "; revision=" + QByteArray::number(revnum) + "\n";
342
343     {
344         QByteArray branchRef = branch;
345         if (!branchRef.startsWith("refs/"))
346             branchRef.prepend("refs/heads/");
347
348         QTextStream s(&repository->fastImport);
349         s << "commit " << branchRef << endl;
350         s << "committer " << QString::fromUtf8(author) << ' ' << datetime << " -0000" << endl;
351
352         Branch &br = repository->branches[branch];
353         if (!br.created) {
354             qWarning() << "Branch" << branch << "in repository" << repository->name << "doesn't exist at revision"
355                        << revnum << "-- did you resume from the wrong revision?";
356             br.created = revnum;
357         }
358
359         s << "data " << message.length() << endl;
360     }
361
362     repository->fastImport.write(message);
363     repository->fastImport.putChar('\n');
364
365     // write the file deletions
366     if (deletedFiles.contains(""))
367         repository->fastImport.write("deleteall\n");
368     else
369         foreach (QString df, deletedFiles)
370             repository->fastImport.write("D " + df.toUtf8() + "\n");
371
372     // write the file modifications
373     repository->fastImport.write(modifiedFiles);
374
375     repository->fastImport.write("\nprogress Commit #" +
376                                  QByteArray::number(repository->commitCount) +
377                                  " branch " + branch +
378                                  " = SVN r" + QByteArray::number(revnum) + "\n\n");
379     printf(" %d modifications to \"%s\"",
380            deletedFiles.count() + modifiedFiles.count(),
381            qPrintable(repository->name));
382
383     while (repository->fastImport.bytesToWrite())
384         if (!repository->fastImport.waitForBytesWritten(-1))
385             qFatal("Failed to write to process: %s", qPrintable(repository->fastImport.errorString()));
386 }