This project has retired. For details please refer to its Attic page.
DatabaseConfig xref
View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.chukwa.database;
20  
21  
22  import org.apache.hadoop.conf.Configuration;
23  import org.apache.hadoop.fs.Path;
24  import java.util.*;
25  import java.io.File;
26  import java.io.FilenameFilter;
27  
28  public class DatabaseConfig {
29    private Configuration config = null;
30    public final static long CENTURY = 36500 * 24 * 60 * 60 * 1000L;
31    public final static long DECADE = 3650 * 24 * 60 * 60 * 1000L;
32    public final static long YEAR = 365 * 24 * 60 * 60 * 1000L;
33    public final static long QUARTER = 91250 * 24 * 60 * 60L;
34    public final static long MONTH = 30 * 24 * 60 * 60 * 1000L;
35    public final static long WEEK = 7 * 24 * 60 * 60 * 1000L;
36    public final static long DAY = 24 * 60 * 60 * 1000L;
37    public final static String MDL_XML = "mdl.xml";
38  
39    public DatabaseConfig(String path) {
40      Path fileResource = new Path(path);
41      config = new Configuration();
42      config.addResource(fileResource);
43    }
44  
45    public DatabaseConfig() {
46      String dataConfig = System.getenv("CHUKWA_CONF_DIR");
47      if (dataConfig == null) {
48        dataConfig = MDL_XML;
49      } else {
50        dataConfig += File.separator + MDL_XML;
51      }
52      Path fileResource = new Path(dataConfig);
53      config = new Configuration();
54      config.addResource(fileResource);
55      
56      if (System.getenv("CHUKWA_CONF_DIR") != null) {
57        // Allow site-specific MDL files to be included in the 
58        // configuration so as to keep the "main" mdl.xml pure.
59        File confDir = new File(System.getenv("CHUKWA_CONF_DIR"));
60        File[] confFiles = confDir.listFiles(new FilenameFilter() {
61  
62          @Override
63          public boolean accept(File dir, String name) {
64            // Implements a naming convention of ending with "mdl.xml"
65            // but is careful not to pick up mdl.xml itself again.
66            return name.endsWith(MDL_XML) && !name.equals(MDL_XML);
67          }
68  
69        });
70  
71        if (confFiles != null) {
72          for (File confFile : confFiles) 
73            config.addResource(new Path(confFile.getAbsolutePath()));
74        }
75      }
76    }
77  
78    public String get(String key) {
79      return config.get(key);
80    }
81  
82    public void put(String key, String value) {
83      this.config.set(key, value);
84    }
85  
86    public Iterator<?> iterator() {
87      return this.config.iterator();
88    }
89  
90    public HashMap<String, String> startWith(String key) {
91      HashMap<String, String> transformer = new HashMap<String, String>();
92      Iterator<?> entries = config.iterator();
93      while (entries.hasNext()) {
94        String entry = entries.next().toString();
95        if (entry.startsWith(key)) {
96          String[] metrics = entry.split("=");
97          transformer.put(metrics[0], metrics[1]);
98        }
99      }
100     return transformer;
101   }
102 
103   public String[] findTableName(String tableName, long start, long end) {
104     String[] tableNames = null;
105     String tableType = "_week";
106     long now = (new Date()).getTime();
107     long timeWindow = end - start;
108     long partitionSize = WEEK;
109     boolean fallback = true;
110 
111     if (config.get("consolidator.table." + tableName) == null) {
112       tableNames = new String[1];
113       tableNames[0] = tableName;
114       return tableNames;
115     }
116     if (timeWindow <= 0) {
117       timeWindow = 1;
118     }
119     if (timeWindow > DECADE) {
120       tableType = "_century";
121       partitionSize = CENTURY;
122     } else if (timeWindow > YEAR) {
123       tableType = "_decade";
124       partitionSize = DECADE;
125     } else if (timeWindow > QUARTER) {
126       tableType = "_year";
127       partitionSize = YEAR;
128     } else if (timeWindow > MONTH) {
129       tableType = "_quarter";
130       partitionSize = QUARTER;
131     } else if (timeWindow > WEEK) {
132       tableType = "_month";
133       partitionSize = MONTH;
134     } else {
135       tableType = "_week";
136       partitionSize = WEEK;
137     }
138 
139     long currentPartition = now / partitionSize;
140     long startPartition = start / partitionSize;
141     long endPartition = end / partitionSize;
142     while (fallback && partitionSize != CENTURY * 100) {
143       // Check if the starting date is in the far distance from current time. If
144       // it is, use down sampled data.
145       if (startPartition + 2 < currentPartition) {
146         fallback = true;
147         if (partitionSize == DAY) {
148           tableType = "_week";
149           partitionSize = WEEK;
150         } else if (partitionSize == WEEK) {
151           tableType = "_month";
152           partitionSize = MONTH;
153         } else if (partitionSize == MONTH) {
154           tableType = "_year";
155           partitionSize = YEAR;
156         } else if (partitionSize == YEAR) {
157           tableType = "_decade";
158           partitionSize = DECADE;
159         } else if (partitionSize == DECADE) {
160           tableType = "_century";
161           partitionSize = CENTURY;
162         } else {
163           partitionSize = 100 * CENTURY;
164         }
165         currentPartition = now / partitionSize;
166         startPartition = start / partitionSize;
167         endPartition = end / partitionSize;
168       } else {
169         fallback = false;
170       }
171     }
172 
173     if (startPartition != endPartition) {
174       int delta = (int) (endPartition - startPartition);
175       tableNames = new String[delta + 1];
176       for (int i = 0; i <= delta; i++) {
177         long partition = startPartition + (long) i;
178         tableNames[i] = tableName + "_" + partition + tableType;
179       }
180     } else {
181       tableNames = new String[1];
182       tableNames[0] = tableName + "_" + startPartition + tableType;
183     }
184     return tableNames;
185   }
186 
187   public String[] findTableNameForCharts(String tableName, long start, long end) {
188     String[] tableNames = null;
189     String tableType = "_week";
190     long now = (new Date()).getTime();
191     long timeWindow = end - start;
192     if (timeWindow > 60 * 60 * 1000) {
193       timeWindow = timeWindow + 1;
194     }
195     long partitionSize = WEEK;
196     boolean fallback = true;
197 
198     if (config.get("consolidator.table." + tableName) == null) {
199       tableNames = new String[1];
200       tableNames[0] = tableName;
201       return tableNames;
202     }
203 
204     if (timeWindow <= 0) {
205       timeWindow = 1;
206     }
207     if (timeWindow > DECADE) {
208       tableType = "_decade";
209       partitionSize = CENTURY;
210     } else if (timeWindow > YEAR) {
211       tableType = "_decade";
212       partitionSize = CENTURY;
213     } else if (timeWindow > QUARTER) {
214       tableType = "_decade";
215       partitionSize = DECADE;
216     } else if (timeWindow > MONTH) {
217       tableType = "_year";
218       partitionSize = YEAR;
219     } else if (timeWindow > WEEK) {
220       tableType = "_quarter";
221       partitionSize = QUARTER;
222     } else if (timeWindow > DAY) {
223       tableType = "_month";
224       partitionSize = MONTH;
225     } else {
226       tableType = "_week";
227       partitionSize = WEEK;
228     }
229 
230 
231     long currentPartition = now / partitionSize;
232     long startPartition = start / partitionSize;
233     long endPartition = end / partitionSize;
234     while (fallback && partitionSize != DECADE * 100) {
235       // Check if the starting date is in the far distance from current time. If
236       // it is, use down sampled data.
237       if (startPartition + 2 < currentPartition) {
238         fallback = true;
239         if (partitionSize == DAY) {
240           tableType = "_month";
241           partitionSize = MONTH;
242         } else if (partitionSize == WEEK) {
243           tableType = "_quarter";
244           partitionSize = QUARTER;
245         } else if (partitionSize == MONTH) {
246           tableType = "_year";
247           partitionSize = YEAR;
248         } else if (partitionSize == YEAR) {
249           tableType = "_decade";
250           partitionSize = DECADE;
251         } else {
252           partitionSize = CENTURY;
253         }
254         currentPartition = now / partitionSize;
255         startPartition = start / partitionSize;
256         endPartition = end / partitionSize;
257       } else {
258         fallback = false;
259       }
260     }
261 
262     if (startPartition != endPartition) {
263       int delta = (int) (endPartition - startPartition);
264       tableNames = new String[delta + 1];
265       for (int i = 0; i <= delta; i++) {
266         long partition = startPartition + (long) i;
267         tableNames[i] = tableName + "_" + partition + tableType;
268       }
269     } else {
270       tableNames = new String[1];
271       tableNames[0] = tableName + "_" + startPartition + tableType;
272     }
273 
274     return tableNames;
275   }
276 
277   public static void main(String[] args) {
278     DatabaseConfig dbc = new DatabaseConfig();
279     String[] names = dbc.findTableName("system_metrics", 1216140020000L,
280         1218645620000L);
281     for (String n : names) {
282       System.out.println("name:" + n);
283     }
284   }
285 }