This project has retired. For details please refer to its Attic page.
TempFileUtil xref
View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.chukwa.util;
20  
21  
22  import java.io.*;
23  import java.util.Calendar;
24  import java.util.Random;
25  import org.apache.hadoop.chukwa.ChukwaArchiveKey;
26  import org.apache.hadoop.chukwa.ChunkImpl;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FSDataOutputStream;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.io.SequenceFile;
32  
33  public class TempFileUtil {
34    public static File makeBinary(int length) throws IOException {
35      File tmpOutput = new File(System.getProperty("test.build.data", "/tmp"),
36          "chukwaTest");
37      FileOutputStream fos = new FileOutputStream(tmpOutput);
38      Random r = new Random();
39      byte[] randomData = new byte[length];
40      r.nextBytes(randomData);
41      randomData[length - 1] = '\n';// need data to end with \n since default
42                                    // tailer uses that
43      fos.write(randomData);
44      fos.flush();
45      fos.close();
46      return tmpOutput;
47    }
48    
49  
50    static class RandSeqFileWriter {
51      java.util.Random r = new java.util.Random();
52      long lastSeqID = 0;
53      
54      public ChunkImpl getARandomChunk() {
55        int ms = r.nextInt(1000);
56        String line = "2008-05-29 10:42:22," + ms
57            + " INFO org.apache.hadoop.dfs.DataNode: Some text goes here"
58            + r.nextInt() + "\n";
59         
60        ChunkImpl c = new ChunkImpl("HadoopLogProcessor", "test",
61        line.length()  + lastSeqID, line.getBytes(), null);
62        lastSeqID += line.length();
63        c.addTag("cluster=\"foocluster\"");
64        return c;
65      }
66    }
67    
68    public static void writeASinkFile(Configuration conf, FileSystem fileSys, Path dest,
69         int chunks) throws IOException {
70       FSDataOutputStream out = fileSys.create(dest);
71  
72       Calendar calendar = Calendar.getInstance();
73       SequenceFile.Writer seqFileWriter = SequenceFile.createWriter(conf, out,
74           ChukwaArchiveKey.class, ChunkImpl.class,
75           SequenceFile.CompressionType.NONE, null);
76       RandSeqFileWriter rw = new RandSeqFileWriter();
77       for (int i = 0; i < chunks; ++i) {
78         ChunkImpl chunk = rw.getARandomChunk();
79         ChukwaArchiveKey archiveKey = new ChukwaArchiveKey();
80  
81         calendar.set(Calendar.YEAR, 2008);
82         calendar.set(Calendar.MONTH, Calendar.MAY);
83         calendar.set(Calendar.DAY_OF_MONTH, 29);
84         calendar.set(Calendar.HOUR, 10);
85         calendar.set(Calendar.MINUTE, 0);
86         calendar.set(Calendar.SECOND, 0);
87         calendar.set(Calendar.MILLISECOND, 0);
88         archiveKey.setTimePartition(calendar.getTimeInMillis());
89         archiveKey.setDataType(chunk.getDataType());
90         archiveKey.setStreamName(chunk.getStreamName());
91         archiveKey.setSeqId(chunk.getSeqID());
92         seqFileWriter.append(archiveKey, chunk);
93       }
94       seqFileWriter.close();
95       out.close();
96     }
97     
98  
99     public static File makeTestFile(String name, int size,File baseDir) throws IOException {
100      File tmpOutput = new File(baseDir, name);
101      FileOutputStream fos = new FileOutputStream(tmpOutput);
102 
103      PrintWriter pw = new PrintWriter(fos);
104      for (int i = 0; i < size; ++i) {
105        pw.print(i + " ");
106        pw.println("abcdefghijklmnopqrstuvwxyz");
107      }
108      pw.flush();
109      pw.close();
110      return tmpOutput;
111    }
112    
113 
114    public static File makeTestFile(String name, int size) throws IOException {
115      return makeTestFile(name, size, new File(System.getProperty("test.build.data", "/tmp")));
116 
117    }
118    
119    public static File makeTestFile(File baseDir) throws IOException {
120      return makeTestFile("atemp",10, baseDir);
121    }
122    
123 
124    public static File makeTestFile() throws IOException {
125      return makeTestFile("atemp",80, new File(System.getProperty("test.build.data", "/tmp")));
126    }
127   
128 }