1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25 package org.apache.hadoop.chukwa.inputtools.log4j;
26
27
28 import java.io.File;
29 import java.io.FilenameFilter;
30 import java.io.IOException;
31 import java.text.SimpleDateFormat;
32 import java.util.ArrayList;
33 import java.util.Calendar;
34 import java.util.Collections;
35 import java.util.Date;
36 import java.util.GregorianCalendar;
37 import java.util.List;
38 import java.util.Locale;
39 import java.util.TimeZone;
40 import java.util.regex.Pattern;
41
42 import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
43 import org.apache.hadoop.chukwa.datacollection.controller.ClientFinalizer;
44 import org.apache.hadoop.chukwa.util.AdaptorNamingUtils;
45 import org.apache.hadoop.chukwa.util.RecordConstants;
46 import org.apache.log4j.FileAppender;
47 import org.apache.log4j.Layout;
48 import org.apache.log4j.Logger;
49 import org.apache.log4j.helpers.LogLog;
50 import org.apache.log4j.spi.LoggingEvent;
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156 public class ChukwaDailyRollingFileAppender extends FileAppender {
157
158 static Logger log = Logger.getLogger(ChukwaDailyRollingFileAppender.class);
159
160
161 static final int TOP_OF_TROUBLE = -1;
162 static final int TOP_OF_MINUTE = 0;
163 static final int TOP_OF_HOUR = 1;
164 static final int HALF_DAY = 2;
165 static final int TOP_OF_DAY = 3;
166 static final int TOP_OF_WEEK = 4;
167 static final int TOP_OF_MONTH = 5;
168
169 static final String adaptorType = ChukwaAgentController.CharFileTailUTF8NewLineEscaped;
170
171 static final Object lock = new Object();
172 static String lastRotation = "";
173
174
175
176
177
178 private String datePattern = "'.'yyyy-MM-dd";
179
180
181
182
183
184
185
186
187
188 private String scheduledFilename;
189
190
191
192
193 private long nextCheck = System.currentTimeMillis() - 1;
194
195
196
197
198 private String cleanUpRegex = null;
199
200
201
202
203 private int maxBackupIndex = 10;
204
205 private ClientFinalizer clientFinalizer = null;
206
207 boolean hasBeenActivated = false;
208 Date now = new Date();
209
210 SimpleDateFormat sdf;
211
212 RollingCalendar rc = new RollingCalendar();
213
214 int checkPeriod = TOP_OF_TROUBLE;
215
216 ChukwaAgentController chukwaClient;
217 boolean chukwaClientIsNull = true;
218 static final Object chukwaLock = new Object();
219
220 String chukwaClientHostname;
221 int chukwaClientPortNum;
222 long chukwaClientConnectNumRetry;
223 long chukwaClientConnectRetryInterval;
224
225 String recordType;
226
227
228 static final TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT");
229
230
231
232
233
234 public ChukwaDailyRollingFileAppender() throws IOException {
235 super();
236 }
237
238
239
240
241
242
243
244
245
246
247
248 public ChukwaDailyRollingFileAppender(Layout layout, String filename,
249 String datePattern) throws IOException {
250 super(layout, filename, true);
251 System.out
252 .println("Daily Rolling File Appender successfully registered file with agent: "
253 + filename);
254 this.datePattern = datePattern;
255 }
256
257
258
259
260
261
262 public void setDatePattern(String pattern) {
263 datePattern = pattern;
264 }
265
266
267
268
269 public String getDatePattern() {
270 return datePattern;
271 }
272
273 public String getRecordType() {
274 if (recordType != null)
275 return recordType;
276 else
277 return "unknown";
278 }
279
280 public void setRecordType(String recordType) {
281 this.recordType = recordType;
282 }
283
284 public void activateOptions() {
285
286
287 if (!hasBeenActivated)
288 { return;}
289
290 super.activateOptions();
291 if (datePattern != null && fileName != null) {
292 now.setTime(System.currentTimeMillis());
293 sdf = new SimpleDateFormat(datePattern);
294 int type = computeCheckPeriod();
295 printPeriodicity(type);
296 rc.setType(type);
297 File file = new File(fileName);
298 scheduledFilename = fileName + sdf.format(new Date(file.lastModified()));
299
300 } else {
301 LogLog
302 .error("Either File or DatePattern options are not set for appender ["
303 + name + "].");
304 }
305 }
306
307 void printPeriodicity(int type) {
308 switch (type) {
309 case TOP_OF_MINUTE:
310 LogLog.debug("Appender [" + name + "] to be rolled every minute.");
311 break;
312 case TOP_OF_HOUR:
313 LogLog
314 .debug("Appender [" + name + "] to be rolled on top of every hour.");
315 break;
316 case HALF_DAY:
317 LogLog.debug("Appender [" + name
318 + "] to be rolled at midday and midnight.");
319 break;
320 case TOP_OF_DAY:
321 LogLog.debug("Appender [" + name + "] to be rolled at midnight.");
322 break;
323 case TOP_OF_WEEK:
324 LogLog.debug("Appender [" + name + "] to be rolled at start of week.");
325 break;
326 case TOP_OF_MONTH:
327 LogLog.debug("Appender [" + name
328 + "] to be rolled at start of every month.");
329 break;
330 default:
331 LogLog.warn("Unknown periodicity for appender [" + name + "].");
332 }
333 }
334
335
336
337
338
339
340
341
342
343
344 int computeCheckPeriod() {
345 RollingCalendar rollingCalendar = new RollingCalendar(gmtTimeZone,
346 Locale.ENGLISH);
347
348 Date epoch = new Date(0);
349 if (datePattern != null) {
350 for (int i = TOP_OF_MINUTE; i <= TOP_OF_MONTH; i++) {
351 SimpleDateFormat simpleDateFormat = new SimpleDateFormat(datePattern);
352 simpleDateFormat.setTimeZone(gmtTimeZone);
353
354 String r0 = simpleDateFormat.format(epoch);
355 rollingCalendar.setType(i);
356 Date next = new Date(rollingCalendar.getNextCheckMillis(epoch));
357 String r1 = simpleDateFormat.format(next);
358
359 if (r0 != null && r1 != null && !r0.equals(r1)) {
360 return i;
361 }
362 }
363 }
364 return TOP_OF_TROUBLE;
365 }
366
367
368
369
370 void rollOver() throws IOException {
371
372
373 if (datePattern == null) {
374 errorHandler.error("Missing DatePattern option in rollOver().");
375 return;
376 }
377
378 String datedFilename = fileName + sdf.format(now);
379
380
381
382 if (scheduledFilename.equals(datedFilename)) {
383 return;
384 }
385
386
387 this.closeFile();
388
389 File target = new File(scheduledFilename);
390 if (target.exists()) {
391 if(!target.delete()) {
392 LogLog.warn("Unable to remove: "+target.getAbsolutePath());
393 };
394 }
395
396 File file = new File(fileName);
397
398 boolean result = file.renameTo(target);
399 if (result) {
400 LogLog.debug(fileName + " -> " + scheduledFilename);
401 } else {
402 LogLog.error("Failed to rename [" + fileName + "] to ["
403 + scheduledFilename + "].");
404 }
405
406 try {
407
408
409 this.setFile(fileName, false, this.bufferedIO, this.bufferSize);
410 } catch (IOException e) {
411 errorHandler.error("setFile(" + fileName + ", false) call failed.");
412 }
413 scheduledFilename = datedFilename;
414 cleanUp();
415 }
416
417 public synchronized String getCleanUpRegex() {
418 return cleanUpRegex;
419 }
420
421 protected synchronized void setCleanUpRegex(String cleanUpRegex) {
422 this.cleanUpRegex = cleanUpRegex;
423 }
424
425 public int getMaxBackupIndex() {
426 return maxBackupIndex;
427 }
428
429 public void setMaxBackupIndex(int maxBackupIndex) {
430 this.maxBackupIndex = maxBackupIndex;
431 }
432
433 protected synchronized void cleanUp() {
434 String regex = "";
435 try {
436 File actualFile = new File(fileName);
437
438 String directoryName = actualFile.getParent();
439 String actualFileName = actualFile.getName();
440 File dirList = new File(directoryName);
441
442 if (cleanUpRegex == null || !cleanUpRegex.contains("$fileName")) {
443 LogLog
444 .error("cleanUpRegex == null || !cleanUpRegex.contains(\"$fileName\")");
445 return;
446 }
447 regex = cleanUpRegex.replace("$fileName", actualFileName);
448 String[] dirFiles = dirList.list(new LogFilter(actualFileName, regex));
449
450 List<String> files = new ArrayList<String>();
451 if(dirFiles!=null) {
452 for (String file : dirFiles) {
453 files.add(file);
454 }
455 }
456 Collections.sort(files);
457
458 while (files.size() > maxBackupIndex) {
459 String file = files.remove(0);
460 File f = new File(directoryName + "/" + file);
461 if(!f.delete()) {
462 LogLog.warn("Cannot remove: " + file);
463 }
464 }
465 } catch (Exception e) {
466 errorHandler
467 .error("cleanUp(" + fileName + "," + regex + ") call failed.");
468 }
469 }
470
471 private static class LogFilter implements FilenameFilter {
472 private Pattern p = null;
473 private String logFile = null;
474
475 public LogFilter(String logFile, String regex) {
476 this.logFile = logFile;
477 p = Pattern.compile(regex);
478 }
479
480 @Override
481 public boolean accept(File dir, String name) {
482
483 if (name.intern() == this.logFile.intern()) {
484 return false;
485 }
486
487 if (!name.startsWith(logFile)) {
488 return false;
489 }
490 return p.matcher(name).find();
491 }
492 }
493
494
495
496
497 @Override
498 protected boolean checkEntryConditions() {
499 synchronized(chukwaLock) {
500 if (!hasBeenActivated) {
501 hasBeenActivated = true;
502 activateOptions();
503 }
504 }
505 return super.checkEntryConditions();
506 }
507
508
509
510
511
512
513
514
515 protected void subAppend(LoggingEvent event) {
516 try {
517
518
519
520
521
522
523
524 if (chukwaClientIsNull) {
525 synchronized (chukwaLock) {
526
527 String log4jFileName = getFile();
528 String recordType = getRecordType();
529
530 long currentLength = 0L;
531 try {
532 File fooLog = new File(log4jFileName);
533 log4jFileName = fooLog.getAbsolutePath();
534 currentLength = fooLog.length();
535 } catch (Throwable e) {
536 log.warn("Exception while trying to get current file size for " + log4jFileName);
537 currentLength = 0L;
538 }
539
540 if (chukwaClient == null) {
541 if (getChukwaClientHostname() != null
542 && getChukwaClientPortNum() != 0) {
543 chukwaClient = new ChukwaAgentController(
544 getChukwaClientHostname(), getChukwaClientPortNum());
545 log.debug("setup adaptor with hostname "
546 + getChukwaClientHostname() + " and portnum "
547 + getChukwaClientPortNum());
548 } else {
549 chukwaClient = new ChukwaAgentController();
550 log
551 .debug("setup adaptor with no args, which means it used its defaults");
552 }
553
554 chukwaClientIsNull = false;
555
556
557
558
559
560
561
562
563 long retryInterval = chukwaClientConnectRetryInterval;
564 if (retryInterval == 0) {
565 retryInterval = 1000 * 60 * 30;
566 }
567 long numRetries = chukwaClientConnectNumRetry;
568 if (numRetries == 0) {
569 numRetries = 48;
570 }
571
572 String name = AdaptorNamingUtils.synthesizeAdaptorID
573 (ChukwaAgentController.CharFileTailUTF8NewLineEscaped, recordType, log4jFileName);
574
575 String adaptorID = chukwaClient.addByName(name, ChukwaAgentController.CharFileTailUTF8NewLineEscaped,
576 recordType,currentLength + " " + log4jFileName, currentLength,
577 numRetries, retryInterval);
578
579
580 clientFinalizer = new ClientFinalizer(chukwaClient);
581 Runtime.getRuntime().addShutdownHook(clientFinalizer);
582
583 if (adaptorID != null) {
584 log.debug("Added file tailing adaptor to chukwa agent for file "
585 + log4jFileName + ", adaptorId:" + adaptorID
586 + " using this recordType :" + recordType
587 + ", starting at offset:" + currentLength);
588 } else {
589 log.debug("Chukwa adaptor not added, addFile(" + log4jFileName
590 + ") returned, current offset: " + currentLength);
591 }
592
593 }
594 }
595 }
596
597 long n = System.currentTimeMillis();
598 if (n >= nextCheck) {
599 now.setTime(n);
600 nextCheck = rc.getNextCheckMillis(now);
601 try {
602 rollOver();
603 } catch (IOException ioe) {
604 LogLog.error("rollOver() failed.", ioe);
605 }
606 }
607
608 boolean written = false;
609 if(layout.ignoresThrowable()) {
610 String[] s = event.getThrowableStrRep();
611 if (s != null) {
612 int len = s.length;
613 StringBuilder sb = new StringBuilder();
614 sb.append(this.layout.format(event));
615 for(int i = 0; i < len; i++) {
616 sb.append(s[i]).append("\n");
617 }
618
619 written = true;
620 this.qw.write(RecordConstants.escapeAllButLastRecordSeparator("\n",sb.toString()));
621 }
622 }
623
624 if (!written) {
625
626 this.qw.write(RecordConstants.escapeAllButLastRecordSeparator("\n",this.layout.format(event)));
627 }
628
629 if (this.immediateFlush) {
630 this.qw.flush();
631 }
632 } catch (Throwable e) {
633 System.err.println("Exception in ChukwaRollingAppender: "
634 + e.getMessage());
635 e.printStackTrace();
636 }
637
638 }
639
640 public String getChukwaClientHostname() {
641 return chukwaClientHostname;
642 }
643
644 public void setChukwaClientHostname(String chukwaClientHostname) {
645 this.chukwaClientHostname = chukwaClientHostname;
646 }
647
648 public int getChukwaClientPortNum() {
649 return chukwaClientPortNum;
650 }
651
652 public void setChukwaClientPortNum(int chukwaClientPortNum) {
653 this.chukwaClientPortNum = chukwaClientPortNum;
654 }
655
656 public void setChukwaClientConnectNumRetry(int i) {
657 this.chukwaClientConnectNumRetry = i;
658 }
659
660 public void setChukwaClientConnectRetryInterval(long i) {
661 this.chukwaClientConnectRetryInterval = i;
662 }
663
664 }
665
666
667
668
669
670
671
672 class RollingCalendar extends GregorianCalendar {
673
674
675
676
677 private static final long serialVersionUID = 2153481574198792767L;
678 int type = ChukwaDailyRollingFileAppender.TOP_OF_TROUBLE;
679
680 RollingCalendar() {
681 super();
682 }
683
684 RollingCalendar(TimeZone tz, Locale locale) {
685 super(tz, locale);
686 }
687
688 void setType(int type) {
689 this.type = type;
690 }
691
692 public long getNextCheckMillis(Date now) {
693 return getNextCheckDate(now).getTime();
694 }
695
696 public Date getNextCheckDate(Date now) {
697 this.setTime(now);
698
699 switch (type) {
700 case ChukwaDailyRollingFileAppender.TOP_OF_MINUTE:
701 this.set(Calendar.SECOND, 0);
702 this.set(Calendar.MILLISECOND, 0);
703 this.add(Calendar.MINUTE, 1);
704 break;
705 case ChukwaDailyRollingFileAppender.TOP_OF_HOUR:
706 this.set(Calendar.MINUTE, 0);
707 this.set(Calendar.SECOND, 0);
708 this.set(Calendar.MILLISECOND, 0);
709 this.add(Calendar.HOUR_OF_DAY, 1);
710 break;
711 case ChukwaDailyRollingFileAppender.HALF_DAY:
712 this.set(Calendar.MINUTE, 0);
713 this.set(Calendar.SECOND, 0);
714 this.set(Calendar.MILLISECOND, 0);
715 int hour = get(Calendar.HOUR_OF_DAY);
716 if (hour < 12) {
717 this.set(Calendar.HOUR_OF_DAY, 12);
718 } else {
719 this.set(Calendar.HOUR_OF_DAY, 0);
720 this.add(Calendar.DAY_OF_MONTH, 1);
721 }
722 break;
723 case ChukwaDailyRollingFileAppender.TOP_OF_DAY:
724 this.set(Calendar.HOUR_OF_DAY, 0);
725 this.set(Calendar.MINUTE, 0);
726 this.set(Calendar.SECOND, 0);
727 this.set(Calendar.MILLISECOND, 0);
728 this.add(Calendar.DATE, 1);
729 break;
730 case ChukwaDailyRollingFileAppender.TOP_OF_WEEK:
731 this.set(Calendar.DAY_OF_WEEK, getFirstDayOfWeek());
732 this.set(Calendar.HOUR_OF_DAY, 0);
733 this.set(Calendar.SECOND, 0);
734 this.set(Calendar.MILLISECOND, 0);
735 this.add(Calendar.WEEK_OF_YEAR, 1);
736 break;
737 case ChukwaDailyRollingFileAppender.TOP_OF_MONTH:
738 this.set(Calendar.DATE, 1);
739 this.set(Calendar.HOUR_OF_DAY, 0);
740 this.set(Calendar.SECOND, 0);
741 this.set(Calendar.MILLISECOND, 0);
742 this.add(Calendar.MONTH, 1);
743 break;
744 default:
745 throw new IllegalStateException("Unknown periodicity type.");
746 }
747 return getTime();
748 }
749
750 @Override
751 public boolean equals(Object o) {
752 return super.equals(o);
753 }
754
755 @Override
756 public int hashCode() {
757 return super.hashCode();
758 }
759 }