This project has retired. For details please refer to its
Attic page.
TempFileUtil xref
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.chukwa.util;
20
21
22 import java.io.*;
23 import java.util.Calendar;
24 import java.util.Random;
25 import org.apache.hadoop.chukwa.ChukwaArchiveKey;
26 import org.apache.hadoop.chukwa.ChunkImpl;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.FSDataOutputStream;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.io.SequenceFile;
32
33 public class TempFileUtil {
34 public static File makeBinary(int length) throws IOException {
35 File tmpOutput = new File(System.getProperty("test.build.data", "/tmp"),
36 "chukwaTest");
37 FileOutputStream fos = new FileOutputStream(tmpOutput);
38 Random r = new Random();
39 byte[] randomData = new byte[length];
40 r.nextBytes(randomData);
41 randomData[length - 1] = '\n';
42
43 fos.write(randomData);
44 fos.flush();
45 fos.close();
46 return tmpOutput;
47 }
48
49
50 static class RandSeqFileWriter {
51 java.util.Random r = new java.util.Random();
52 long lastSeqID = 0;
53
54 public ChunkImpl getARandomChunk() {
55 int ms = r.nextInt(1000);
56 String line = "2008-05-29 10:42:22," + ms
57 + " INFO org.apache.hadoop.dfs.DataNode: Some text goes here"
58 + r.nextInt() + "\n";
59
60 ChunkImpl c = new ChunkImpl("HadoopLogProcessor", "test",
61 line.length() + lastSeqID, line.getBytes(), null);
62 lastSeqID += line.length();
63 c.addTag("cluster=\"foocluster\"");
64 return c;
65 }
66 }
67
68 public static void writeASinkFile(Configuration conf, FileSystem fileSys, Path dest,
69 int chunks) throws IOException {
70 FSDataOutputStream out = fileSys.create(dest);
71
72 Calendar calendar = Calendar.getInstance();
73 SequenceFile.Writer seqFileWriter = SequenceFile.createWriter(conf, out,
74 ChukwaArchiveKey.class, ChunkImpl.class,
75 SequenceFile.CompressionType.NONE, null);
76 RandSeqFileWriter rw = new RandSeqFileWriter();
77 for (int i = 0; i < chunks; ++i) {
78 ChunkImpl chunk = rw.getARandomChunk();
79 ChukwaArchiveKey archiveKey = new ChukwaArchiveKey();
80
81 calendar.set(Calendar.YEAR, 2008);
82 calendar.set(Calendar.MONTH, Calendar.MAY);
83 calendar.set(Calendar.DAY_OF_MONTH, 29);
84 calendar.set(Calendar.HOUR, 10);
85 calendar.set(Calendar.MINUTE, 0);
86 calendar.set(Calendar.SECOND, 0);
87 calendar.set(Calendar.MILLISECOND, 0);
88 archiveKey.setTimePartition(calendar.getTimeInMillis());
89 archiveKey.setDataType(chunk.getDataType());
90 archiveKey.setStreamName(chunk.getStreamName());
91 archiveKey.setSeqId(chunk.getSeqID());
92 seqFileWriter.append(archiveKey, chunk);
93 }
94 seqFileWriter.close();
95 out.close();
96 }
97
98
99 public static File makeTestFile(String name, int size,File baseDir) throws IOException {
100 File tmpOutput = new File(baseDir, name);
101 FileOutputStream fos = new FileOutputStream(tmpOutput);
102
103 PrintWriter pw = new PrintWriter(fos);
104 for (int i = 0; i < size; ++i) {
105 pw.print(i + " ");
106 pw.println("abcdefghijklmnopqrstuvwxyz");
107 }
108 pw.flush();
109 pw.close();
110 return tmpOutput;
111 }
112
113
114 public static File makeTestFile(String name, int size) throws IOException {
115 return makeTestFile(name, size, new File(System.getProperty("test.build.data", "/tmp")));
116
117 }
118
119 public static File makeTestFile(File baseDir) throws IOException {
120 return makeTestFile("atemp",10, baseDir);
121 }
122
123
124 public static File makeTestFile() throws IOException {
125 return makeTestFile("atemp",80, new File(System.getProperty("test.build.data", "/tmp")));
126 }
127
128 }