This project has retired. For details please refer to its Attic page.
ChukwaHBaseStore xref
View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.chukwa.datastore;
19  
20  import java.io.IOException;
21  import java.net.InetAddress;
22  import java.net.URI;
23  import java.net.URISyntaxException;
24  import java.nio.ByteBuffer;
25  import java.nio.charset.Charset;
26  import java.util.ArrayList;
27  import java.util.Calendar;
28  import java.util.HashMap;
29  import java.util.HashSet;
30  import java.util.Iterator;
31  import java.util.List;
32  import java.util.Set;
33  import java.util.TimeZone;
34  import java.util.UUID;
35  import java.util.concurrent.CopyOnWriteArraySet;
36  import java.util.concurrent.TimeUnit;
37  
38  import org.apache.hadoop.chukwa.hicc.bean.Chart;
39  import org.apache.hadoop.chukwa.hicc.bean.Dashboard;
40  import org.apache.hadoop.chukwa.hicc.bean.HeatMapPoint;
41  import org.apache.hadoop.chukwa.hicc.bean.Heatmap;
42  import org.apache.hadoop.chukwa.hicc.bean.LineOptions;
43  import org.apache.hadoop.chukwa.hicc.bean.Series;
44  import org.apache.hadoop.chukwa.hicc.bean.SeriesMetaData;
45  import org.apache.hadoop.chukwa.hicc.bean.Widget;
46  import org.apache.hadoop.chukwa.util.ExceptionUtil;
47  import org.apache.hadoop.chukwa.util.HBaseUtil;
48  import org.apache.hadoop.hbase.Cell;
49  import org.apache.hadoop.hbase.CellUtil;
50  import org.apache.hadoop.hbase.TableName;
51  import org.apache.hadoop.hbase.client.Connection;
52  import org.apache.hadoop.hbase.client.ConnectionFactory;
53  import org.apache.hadoop.hbase.client.Delete;
54  import org.apache.hadoop.hbase.client.Get;
55  import org.apache.hadoop.hbase.client.Put;
56  import org.apache.hadoop.hbase.client.Result;
57  import org.apache.hadoop.hbase.client.ResultScanner;
58  import org.apache.hadoop.hbase.client.Scan;
59  import org.apache.hadoop.hbase.client.Table;
60  import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
61  import org.apache.hadoop.hbase.filter.Filter;
62  import org.apache.hadoop.hbase.util.Bytes;
63  import org.apache.log4j.Logger;
64  import org.json.simple.JSONObject;
65  import org.json.simple.JSONValue;
66  
67  import com.google.gson.Gson;
68  
69  public class ChukwaHBaseStore {
70    static Logger LOG = Logger.getLogger(ChukwaHBaseStore.class);
71    static int MINUTES_IN_HOUR = 60;
72    static double RESOLUTION = 360;
73    static int MINUTE = 60000; //60 milliseconds
74    final static int SECOND = (int) TimeUnit.SECONDS.toMillis(1);
75    private final static Charset UTF8 = Charset.forName("UTF-8");
76  
77    final static byte[] COLUMN_FAMILY = "t".getBytes(UTF8);
78    final static byte[] ANNOTATION_FAMILY = "a".getBytes(UTF8);
79    final static byte[] KEY_NAMES = "k".getBytes(UTF8);
80    final static byte[] CHART_TYPE = "chart_meta".getBytes(UTF8);
81    final static byte[] CHART_FAMILY = "c".getBytes(UTF8);
82    final static byte[] COMMON_FAMILY = "c".getBytes(UTF8);
83    final static byte[] WIDGET_TYPE = "widget_meta".getBytes(UTF8);
84    final static byte[] DASHBOARD_TYPE = "dashboard_meta".getBytes(UTF8);
85    private static final String CHUKWA = "chukwa";
86    private static final String CHUKWA_META = "chukwa_meta";
87    private static long MILLISECONDS_IN_DAY = 86400000L;
88    private static Connection connection = null;
89  
90    public ChukwaHBaseStore() {
91      super();
92    }
93  
94    public static synchronized void getHBaseConnection() throws IOException {
95      if (connection == null || connection.isClosed()) {
96        connection = ConnectionFactory.createConnection();
97      }
98    }
99    
100   public static synchronized void closeHBase() {
101     try {
102       if(connection != null) {
103         connection.close();
104       }
105     } catch(IOException e) {
106       LOG.warn("Unable to release HBase connection.");
107     }
108   }
109   
110   /**
111    * Scan chukwa table for a particular metric group and metric name based on
112    * time ranges.
113    * 
114    * @param metricGroup
115    * @param metric
116    * @param source
117    * @param startTime
118    * @param endTime
119    * @return Series
120    */
121   public static Series getSeries(String metricGroup, String metric,
122       String source, long startTime, long endTime) {
123     String fullMetricName = new StringBuilder(metricGroup).append(".")
124         .append(metric).toString();
125     return getSeries(fullMetricName, source, startTime, endTime);
126   }
127 
128   /**
129    * Scan chukwa table for a full metric name based on time ranges.
130    * 
131    * @param metric
132    * @param source
133    * @param startTime
134    * @param endTime
135    * @return Series
136    */
137   public static synchronized Series getSeries(String metric, String source, long startTime,
138       long endTime) {
139     String seriesName = new StringBuilder(metric).append(":").append(source).toString();
140     Series series = new Series(seriesName);
141     try {
142       // Swap start and end if the values are inverted.
143       if (startTime > endTime) {
144         long temp = endTime;
145         startTime = endTime;
146         endTime = temp;
147       }
148 
149       getHBaseConnection();
150       Table table = connection.getTable(TableName.valueOf(CHUKWA));
151       Scan scan = new Scan();
152       Calendar c = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
153       c.setTimeInMillis(startTime);
154       int startDay = c.get(Calendar.DAY_OF_YEAR);
155       c.setTimeInMillis(endTime);
156       int endDay = c.get(Calendar.DAY_OF_YEAR);
157       long currentDay = startTime;
158       for (int i = startDay; i <= endDay; i++) {
159         byte[] rowKey = HBaseUtil.buildKey(currentDay, metric, source);
160 
161         scan.addFamily(COLUMN_FAMILY);
162         scan.setStartRow(rowKey);
163         scan.setStopRow(rowKey);
164         scan.setTimeRange(startTime, endTime);
165         scan.setBatch(10000);
166 
167         ResultScanner results = table.getScanner(scan);
168         Iterator<Result> it = results.iterator();
169 
170         while (it.hasNext()) {
171           Result result = it.next();
172           for (Cell kv : result.rawCells()) {
173             byte[] key = CellUtil.cloneQualifier(kv);
174             long timestamp = ByteBuffer.wrap(key).getLong();
175             double value = Double
176                 .parseDouble(new String(CellUtil.cloneValue(kv), UTF8));
177             series.add(timestamp, value);
178           }
179         }
180         results.close();
181         currentDay = currentDay + (i * MILLISECONDS_IN_DAY);
182       }
183       table.close();
184     } catch (IOException e) {
185       closeHBase();
186       LOG.error(ExceptionUtil.getStackTrace(e));
187     }
188     return series;
189   }
190 
191   public static Set<String> getMetricNames(String metricGroup) {
192     Set<String> familyNames = new CopyOnWriteArraySet<String>();
193     try {
194       getHBaseConnection();
195       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
196       Get get = new Get(metricGroup.getBytes(UTF8));
197       Result result = table.get(get);
198       for (Cell kv : result.rawCells()) {
199         JSONObject json = (JSONObject) JSONValue.parse(new String(CellUtil.cloneValue(kv), UTF8));
200         if (json.get("type").equals("metric")) {
201           familyNames.add(new String(CellUtil.cloneQualifier(kv), UTF8));
202         }
203       }
204       table.close();
205     } catch (Exception e) {
206       closeHBase();
207       LOG.error(ExceptionUtil.getStackTrace(e));
208     }
209     return familyNames;
210 
211   }
212 
213   public static Set<String> getMetricGroups() {
214     Set<String> metricGroups = new CopyOnWriteArraySet<String>();
215     try {
216       getHBaseConnection();
217       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
218       Scan scan = new Scan();
219       scan.addFamily(KEY_NAMES);
220       ResultScanner rs = table.getScanner(scan);
221       Iterator<Result> it = rs.iterator();
222       while (it.hasNext()) {
223         Result result = it.next();
224         metricGroups.add(new String(result.getRow(), UTF8));
225       }
226       table.close();
227     } catch (Exception e) {
228       closeHBase();
229       LOG.error(ExceptionUtil.getStackTrace(e));
230     }
231     return metricGroups;
232   }
233 
234   public static Set<String> getSourceNames(String dataType) {
235     Set<String> pk = new HashSet<String>();
236     try {
237       getHBaseConnection();
238       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
239       Scan scan = new Scan();
240       scan.addFamily(KEY_NAMES);
241       ResultScanner rs = table.getScanner(scan);
242       Iterator<Result> it = rs.iterator();
243       while (it.hasNext()) {
244         Result result = it.next();
245         for (Cell cell : result.rawCells()) {
246           JSONObject json = (JSONObject) JSONValue.parse(new String(CellUtil.cloneValue(cell), UTF8));
247           if (json!=null && json.get("type")!=null && json.get("type").equals("source")) {
248             pk.add(new String(CellUtil.cloneQualifier(cell), UTF8));
249           }
250         }
251       }
252       rs.close();
253       table.close();
254     } catch (Exception e) {
255       closeHBase();
256       LOG.error(ExceptionUtil.getStackTrace(e));
257     }
258     return pk;
259   }
260 
261   public static Heatmap getHeatmap(String metricGroup, String metric,
262       long startTime, long endTime, double max, double scale, int width, int height) {
263     Heatmap heatmap = new Heatmap();
264     Set<String> sources = getSourceNames(metricGroup);
265     Set<String> metrics = getMetricNames(metricGroup);
266     List<Get> series = new ArrayList<Get>();
267     String fullName = new StringBuilder(metricGroup).append(".").append(metric).toString();
268     try {
269       getHBaseConnection();
270       Table table = connection.getTable(TableName.valueOf(CHUKWA));
271       Calendar c = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
272       c.setTimeInMillis(startTime);
273       int startDay = c.get(Calendar.DAY_OF_YEAR);
274       c.setTimeInMillis(endTime);
275       int endDay = c.get(Calendar.DAY_OF_YEAR);
276       long currentDay = startTime;
277       for (int i = startDay; i <= endDay; i++) {
278         for (String m : metrics) {
279           if (m.startsWith(fullName)) {
280             for (String source : sources) {
281               byte[] rowKey = HBaseUtil.buildKey(currentDay, m, source);
282               Get serie = new Get(rowKey);
283               serie.addFamily(COLUMN_FAMILY);
284               serie.setTimeRange(startTime, endTime);
285               series.add(serie);
286             }
287           }
288         }
289         currentDay = currentDay + (i * MILLISECONDS_IN_DAY);
290       }
291       long timeRange = (endTime - startTime);
292       Result[] rs = table.get(series);
293       int index = 1;
294       // Series display in y axis
295       int y = 0;
296       HashMap<String, Integer> keyMap = new HashMap<String, Integer>();
297       for (Result result : rs) {
298         for(Cell cell : result.rawCells()) {
299           byte[] dest = new byte[5];
300           System.arraycopy(CellUtil.cloneRow(cell), 3, dest, 0, 5);
301           String source = new String(dest, UTF8);
302           long time = cell.getTimestamp();
303           // Time display in x axis
304           long delta = time - startTime;
305           double f = (double) delta / timeRange;
306           f = (double) f * width;
307           int x = (int) Math.round(f);
308           if (keyMap.containsKey(source)) {
309             y = keyMap.get(source);
310           } else {
311             keyMap.put(source, Integer.valueOf(index));
312             y = index;
313             index++;
314           }
315           double v = Double.parseDouble(new String(CellUtil.cloneValue(cell), UTF8));
316           heatmap.put(x, y, v);
317           if (v > max) {
318             max = v;
319           }
320         }
321       }
322       table.close();
323       int radius = height / index;
324       // Usually scale max from 0 to 100 for visualization
325       heatmap.putMax(scale);
326       for (HeatMapPoint point : heatmap.getHeatmap()) {
327         double round = point.count / max * scale;
328         round = Math.round(round * 100.0) / 100.0;
329         point.put(point.x, point.y * radius, round);
330       }
331       heatmap.putRadius(radius);
332       heatmap.putSeries(index -1);
333     } catch (IOException e) {
334       closeHBase();
335       LOG.error(ExceptionUtil.getStackTrace(e));
336     }
337     return heatmap;
338   }
339 
340   /**
341    * Scan chukwa table and find cluster tag from annotation column family from a
342    * range of entries.
343    * 
344    * @param startTime
345    * @param endTime
346    * @return Set<String> of cluster names
347    */
348   public static Set<String> getClusterNames(long startTime, long endTime) {
349     Set<String> clusters = new HashSet<String>();
350     try {
351       getHBaseConnection();
352       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
353       Scan scan = new Scan();
354       scan.addFamily(KEY_NAMES);
355       ResultScanner rs = table.getScanner(scan);
356       Iterator<Result> it = rs.iterator();
357       while (it.hasNext()) {
358         Result result = it.next();
359         for (Cell cell : result.rawCells()) {
360           JSONObject json = (JSONObject) JSONValue.parse(new String(CellUtil.cloneValue(cell), UTF8));
361           if (json.get("type").equals("cluster")) {
362             clusters.add(new String(CellUtil.cloneQualifier(cell), UTF8));
363           }
364         }
365       }
366       table.close();
367     } catch (Exception e) {
368       closeHBase();
369       LOG.error(ExceptionUtil.getStackTrace(e));
370     }
371     return clusters;
372   }
373 
374   /**
375    * Get a chart from HBase by ID.
376    * 
377    * @param id
378    * @return Chart
379    */
380   public static Chart getChart(String id) {
381     Chart chart = null;
382     try {
383       getHBaseConnection();
384       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
385       Get get = new Get(CHART_TYPE);
386       Result r = table.get(get);
387       byte[] value = r.getValue(CHART_FAMILY, id.getBytes(UTF8));
388       Gson gson = new Gson();
389       if(value!=null) {
390         chart = gson.fromJson(new String(value, UTF8), Chart.class);
391       }
392       table.close();
393     } catch (Exception e) {
394       closeHBase();
395       LOG.error(ExceptionUtil.getStackTrace(e));
396     }
397     return chart;
398   }
399 
400   /**
401    * Update a chart in HBase by ID.
402    * 
403    * @param id
404    * @param chart
405    */
406   public static void putChart(String id, Chart chart) {
407     try {
408       getHBaseConnection();
409       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
410       Put put = new Put(CHART_TYPE);
411       Gson gson = new Gson();
412       String buffer = gson.toJson(chart);
413       put.addColumn(CHART_FAMILY, id.getBytes(UTF8), buffer.getBytes(UTF8));
414       table.put(put);
415       table.close();
416     } catch (Exception e) {
417       closeHBase();
418       LOG.error(ExceptionUtil.getStackTrace(e));
419     }
420     
421   }
422 
423   /**
424    * Create a chart in HBase by specifying parameters.
425    * @param id 
426    * @param title 
427    * @param metrics 
428    * @param source 
429    * @param yunitType 
430    * @return 
431    * @throws URISyntaxException 
432    */
433   public static synchronized String createChart(String id,
434       String title, String[] metrics, String source, String yunitType) throws URISyntaxException {
435     Chart chart = new Chart(id);
436     chart.setYUnitType(yunitType);
437     chart.setTitle(title);
438     ArrayList<SeriesMetaData> series = new ArrayList<SeriesMetaData>();
439     for(String metric : metrics) {
440       SeriesMetaData s = new SeriesMetaData();
441       s.setLabel(metric + "/" + source);
442       s.setUrl(new URI("/hicc/v1/metrics/series/" + metric + "/"
443         + source));
444       LineOptions l = new LineOptions();
445       s.setLineOptions(l);
446       series.add(s);
447     }
448     chart.setSeries(series);
449     return createChart(chart);
450     
451   }
452 
453   /**
454    * Create a chart in HBase by specifying parameters.
455    * @param id 
456    * @param title 
457    * @param metrics 
458    * @param source 
459    * @param suffixLabel 
460    * @param direction 
461    * @return 
462    * @throws URISyntaxException 
463    */
464   public static synchronized String createCircle(String id,
465       String title, String[] metrics, String source, String suffixLabel, String direction) throws URISyntaxException {
466     Chart chart = new Chart(id);
467     chart.setSuffixText(suffixLabel);
468     chart.setTitle(title);
469     chart.setThreshold(direction);
470     ArrayList<SeriesMetaData> series = new ArrayList<SeriesMetaData>();
471     for(String metric : metrics) {
472       SeriesMetaData s = new SeriesMetaData();
473       s.setLabel(metric + "/" + source);
474       s.setUrl(new URI("/hicc/v1/metrics/series/" + metric + "/"
475         + source));
476       series.add(s);
477     }
478     chart.setSeries(series);
479     return createChart(chart);
480     
481   }
482 
483   /**
484    * Create a tile in HBase by specifying parameters.
485    * @param id
486    * @param title
487    * @param bannerText
488    * @param suffixLabel
489    * @param metrics
490    * @param source
491    * @param icon
492    * @return html
493    * @throws URISyntaxException
494    */
495   public static synchronized String createTile(String id, String title, 
496       String bannerText, String suffixLabel, String[] metrics, String source, 
497       String icon) throws URISyntaxException {
498     Chart chart = new Chart(id);
499     chart.setTitle(title);
500     chart.setBannerText(bannerText);
501     chart.setSuffixText(suffixLabel);
502     chart.setIcon(icon);
503     List<SeriesMetaData> smd = new ArrayList<SeriesMetaData>();
504     for (String metric : metrics) {
505       SeriesMetaData series = new SeriesMetaData();
506       series.setUrl(new URI("/hicc/v1/metrics/series/" + metric + "/" + source));
507       smd.add(series);
508     }
509     chart.setSeries(smd);
510     return createChart(chart);
511   }
512 
513   /**
514    * Create a chart in HBase.
515    * 
516    * @param chart
517    * @return id of newly created chart
518    */
519   public static synchronized String createChart(Chart chart) {
520     String id = chart.getId();
521     try {
522       getHBaseConnection();
523       if (id != null) {
524         // Check if there is existing chart with same id.
525         Chart test = getChart(id);
526         if (test != null) {
527           // If id already exists, randomly generate an id.
528           id = String.valueOf(UUID.randomUUID());
529         }
530       } else {
531         // If id is not provided, randomly generate an id.
532         id = String.valueOf(UUID.randomUUID());
533       }
534       chart.setId(id);
535       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
536       Put put = new Put(CHART_TYPE);
537       Gson gson = new Gson();
538       String buffer = gson.toJson(chart);
539       put.addColumn(CHART_FAMILY, id.getBytes(UTF8), buffer.getBytes(UTF8));
540       table.put(put);
541       table.close();
542     } catch (Exception e) {
543       closeHBase();
544       LOG.error(ExceptionUtil.getStackTrace(e));
545       id = null;
546     }
547     return id;
548   }
549 
550   /**
551    * Return data for multiple series of metrics stored in HBase.
552    * 
553    * @param series
554    * @param startTime
555    * @param endTime
556    * @return List of series meta data
557    */
558   public static synchronized ArrayList<org.apache.hadoop.chukwa.hicc.bean.SeriesMetaData> getChartSeries(ArrayList<org.apache.hadoop.chukwa.hicc.bean.SeriesMetaData> series, long startTime, long endTime) {
559     ArrayList<org.apache.hadoop.chukwa.hicc.bean.SeriesMetaData> list = new ArrayList<org.apache.hadoop.chukwa.hicc.bean.SeriesMetaData>();
560     try {
561       // Swap start and end if the values are inverted.
562       if (startTime > endTime) {
563         long temp = endTime;
564         startTime = endTime;
565         endTime = temp;
566       }
567       // Figure out the time range and determine the best resolution
568       // to fetch the data
569       long range = (endTime - startTime)
570         / (long) (MINUTES_IN_HOUR * MINUTE);
571       long sampleRate = 1;
572       if (range <= 1) {
573         sampleRate = 5;
574       } else if(range <= 24) {
575         sampleRate = 240;
576       } else if (range <= 720) {
577         sampleRate = 7200;
578       } else if(range >= 720) {
579         sampleRate = 87600;
580       }
581       double smoothing = (endTime - startTime)
582           / (double) (sampleRate * SECOND ) / (double) RESOLUTION;
583 
584       getHBaseConnection();
585       Table table = connection.getTable(TableName.valueOf(CHUKWA));
586       Scan scan = new Scan();
587       Calendar c = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
588       c.setTimeInMillis(startTime);
589       int startDay = c.get(Calendar.DAY_OF_YEAR);
590       c.setTimeInMillis(endTime);
591       int endDay = c.get(Calendar.DAY_OF_YEAR);
592       for (org.apache.hadoop.chukwa.hicc.bean.SeriesMetaData s : series) {
593         org.apache.hadoop.chukwa.hicc.bean.SeriesMetaData clone = (org.apache.hadoop.chukwa.hicc.bean.SeriesMetaData) s.clone();
594         long currentDay = startTime;
595         String[] parts = s.getUrl().toString().split("/");
596         String metric = parts[5];
597         String source = parts[6];
598         ArrayList<ArrayList<Number>> data = new ArrayList<ArrayList<Number>>();
599         for (int i = startDay; i <= endDay; i++) {
600           byte[] rowKey = HBaseUtil.buildKey(currentDay, metric, source);
601           scan.addFamily(COLUMN_FAMILY);
602           scan.setStartRow(rowKey);
603           scan.setStopRow(rowKey);
604           scan.setTimeRange(startTime, endTime);
605           scan.setBatch(10000);
606 
607           ResultScanner results = table.getScanner(scan);
608           Iterator<Result> it = results.iterator();
609           double filteredValue = 0.0d;
610           long lastTime = startTime;
611           long totalElapsedTime = 0;
612           int initial = 0;
613           
614           while (it.hasNext()) {
615             Result result = it.next();
616             for (Cell kv : result.rawCells()) {
617               byte[] key = CellUtil.cloneQualifier(kv);
618               long timestamp = ByteBuffer.wrap(key).getLong();
619               double value = Double.parseDouble(new String(CellUtil.cloneValue(kv),
620                   UTF8));
621               if(initial==0) {
622                 filteredValue = value;
623               }
624               long elapsedTime = (timestamp - lastTime) / SECOND;
625               lastTime = timestamp;
626               // Determine if there is any gap, if there is gap in data, reset
627               // calculation.
628               if (elapsedTime > (sampleRate * 5)) {
629                 filteredValue = 0.0d;
630               } else {
631                 if (smoothing != 0.0d) {
632                   // Apply low pass filter to calculate
633                   filteredValue = filteredValue + (double) ((double) elapsedTime * (double) ((double) (value - filteredValue) / smoothing));
634                 } else {
635                   // Use original value
636                   filteredValue = value;
637                 }
638               }
639               totalElapsedTime = totalElapsedTime + elapsedTime;
640               if (totalElapsedTime >= sampleRate) {
641                 ArrayList<Number> points = new ArrayList<Number>();
642                 points.add(timestamp);
643                 points.add(filteredValue);
644                 data.add(points);
645                 totalElapsedTime = 0;
646               }
647             }
648             initial++;
649           }
650           results.close();
651           currentDay = currentDay + (i * MILLISECONDS_IN_DAY);
652         }
653         clone.setData(data);
654         list.add(clone);
655       }
656       table.close();
657     } catch (IOException|CloneNotSupportedException e) {
658       closeHBase();
659       LOG.error(ExceptionUtil.getStackTrace(e));
660     }
661     return list;
662   }
663 
664   /**
665    * List widgets stored in HBase.
666    * 
667    * @param limit
668    * @param offset
669    * @return List of widgets
670    */
671   public static synchronized List<Widget> listWidget(int limit, int offset) {
672     ArrayList<Widget> list = new ArrayList<Widget>();
673     try {
674       getHBaseConnection();
675       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
676       Scan scan = new Scan();
677       scan.setStartRow(WIDGET_TYPE);
678       scan.setStopRow(WIDGET_TYPE);
679       ResultScanner rs = table.getScanner(scan);
680       Iterator<Result> it = rs.iterator();
681       int c = 0;
682       while(it.hasNext()) {
683         Result result = it.next();
684         for(Cell kv : result.rawCells()) {
685           if(c > limit) {
686             break;
687           }
688           if(c < offset) {
689             continue;
690           }
691           Gson gson = new Gson();
692           Widget widget = gson.fromJson(new String(CellUtil.cloneValue(kv), UTF8), Widget.class);
693           list.add(widget);
694           c++;
695         }
696       }
697       rs.close();
698       table.close();
699     } catch (Exception e) {
700       closeHBase();
701       LOG.error(ExceptionUtil.getStackTrace(e));
702     }
703     return list;
704   }
705 
706   public static synchronized List<String> getData(ArrayList<SeriesMetaData> series, long startTime, long endTime) {
707     ArrayList<String> data = new ArrayList<String>();
708     try {
709       getHBaseConnection();
710       Table table = connection.getTable(TableName.valueOf(CHUKWA));
711       Scan scan = new Scan();
712       for(SeriesMetaData s : series) {
713         String snapshot = "";
714         String[] parts = s.getUrl().toString().split("/");
715         String metric = parts[5];
716         String source = parts[6];
717         long currentDay = startTime;
718         byte[] rowKey = HBaseUtil.buildKey(currentDay, metric, source);
719         scan.addFamily(COLUMN_FAMILY);
720         scan.setStartRow(rowKey);
721         scan.setStopRow(rowKey);
722         scan.setTimeRange(startTime, endTime);
723         scan.setBatch(10000);
724         ResultScanner rs = table.getScanner(scan);
725         Iterator<Result> it = rs.iterator();
726         while(it.hasNext()) {
727           Result result = it.next();
728           for(Cell kv : result.rawCells()) {
729             snapshot = new String(CellUtil.cloneValue(kv));
730             if(snapshot.matches("-?\\d+(\\.\\d+)?")) {
731               int endOffset = snapshot.length();
732               if(snapshot.length() - snapshot.indexOf(".") > 2) {
733                 endOffset = snapshot.indexOf(".") + 2;
734               }
735               snapshot = snapshot.substring(0, endOffset);
736             }
737           }
738         }
739         data.add(snapshot);
740         rs.close();
741       }
742       table.close();
743     } catch(Exception e) {
744       closeHBase();
745       LOG.error(ExceptionUtil.getStackTrace(e));
746     }
747     return data;
748   }
749 
750   /**
751    * Find widget by title prefix in HBase.
752    * 
753    * @param query - Prefix query of widget title.
754    * @return List of widgets
755    */
756   public static synchronized List<Widget> searchWidget(String query) {
757     ArrayList<Widget> list = new ArrayList<Widget>();
758     try {
759       getHBaseConnection();
760       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
761       Filter filter = new ColumnPrefixFilter(Bytes.toBytes(query));
762       Scan scan = new Scan();
763       scan.setStartRow(WIDGET_TYPE);
764       scan.setStopRow(WIDGET_TYPE);
765       scan.setFilter(filter);
766       ResultScanner rs = table.getScanner(scan);
767       Iterator<Result> it = rs.iterator();
768       while(it.hasNext()) {
769         Result result = it.next();
770         for(Cell kv : result.rawCells()) {
771           Gson gson = new Gson();
772           Widget widget = gson.fromJson(new String(CellUtil.cloneValue(kv), UTF8), Widget.class);
773           list.add(widget);
774         }
775       }
776       rs.close();
777       table.close();
778     } catch (Exception e) {
779       closeHBase();
780       LOG.error(ExceptionUtil.getStackTrace(e));
781     }
782     return list;
783   }
784 
785   /**
786    * View a widget information in HBase.
787    * 
788    * @param title - Title of the widget.
789    * @return Widget
790    */
791   public static synchronized Widget viewWidget(String title) {
792     Widget w = null;
793     try {
794       getHBaseConnection();
795       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
796       Get widget = new Get(WIDGET_TYPE);
797       widget.addColumn(COMMON_FAMILY, title.getBytes(UTF8));
798       Result rs = table.get(widget);
799       byte[] buffer = rs.getValue(COMMON_FAMILY, title.getBytes(UTF8));
800       Gson gson = new Gson();
801       w = gson.fromJson(new String(buffer, UTF8), Widget.class);
802       table.close();
803     } catch (Exception e) {
804       closeHBase();
805       LOG.error(ExceptionUtil.getStackTrace(e));
806     }
807     return w;
808   }
809 
810   /**
811    * Create a widget in HBase.
812    * 
813    * @param widget
814    * @return 
815    */
816   public static synchronized boolean createWidget(Widget widget) {
817     boolean created = false;
818     try {
819       widget.tokenize();
820       getHBaseConnection();
821       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
822       Get widgetTest = new Get(WIDGET_TYPE);
823       widgetTest.addColumn(COMMON_FAMILY, widget.getTitle().getBytes(UTF8));
824       if (table.exists(widgetTest)) {
825         LOG.warn("Widget: " + widget.getTitle() + " already exists.");
826         created = false;
827       } else {
828         Put put = new Put(WIDGET_TYPE);
829         Gson gson = new Gson();
830         String buffer = gson.toJson(widget);
831         put.addColumn(COMMON_FAMILY, widget.getTitle().getBytes(UTF8), buffer.getBytes(UTF8));
832         table.put(put);
833         created = true;
834       }
835       table.close();
836     } catch (Exception e) {
837       closeHBase();
838       LOG.error(ExceptionUtil.getStackTrace(e));
839     }
840     return created;
841   }
842 
843   /**
844    * Update a widget in HBase.
845    * 
846    * @param title
847    * @param widget
848    * @return 
849    */
850   public static synchronized boolean updateWidget(String title, Widget widget) {
851     boolean result = false;
852     try {
853       getHBaseConnection();
854       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
855       Delete oldWidget = new Delete(WIDGET_TYPE);
856       oldWidget.addColumn(COMMON_FAMILY, title.getBytes(UTF8));
857       table.delete(oldWidget);
858       Put put = new Put(WIDGET_TYPE);
859       Gson gson = new Gson();
860       String buffer = gson.toJson(widget);
861       put.addColumn(COMMON_FAMILY, title.getBytes(UTF8), buffer.getBytes(UTF8));
862       table.put(put);
863       table.close();
864       result = true;
865     } catch (Exception e) {
866       closeHBase();
867       LOG.error(ExceptionUtil.getStackTrace(e));
868       LOG.error("Error in updating widget, original title: " + 
869         title + " new title:" + widget.getTitle());
870     }
871     return result;
872   }
873 
874   /**
875    * Delete a widget in HBase.
876    * 
877    * @param title
878    * @return 
879    */
880   public static synchronized boolean deleteWidget(String title) {
881     boolean result = false;
882     try {
883       getHBaseConnection();
884       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
885       Delete oldWidget = new Delete(WIDGET_TYPE);
886       oldWidget.addColumn(COMMON_FAMILY, title.getBytes(UTF8));
887       table.delete(oldWidget);
888       table.close();
889       result = true;
890     } catch (Exception e) {
891       closeHBase();
892       LOG.error(ExceptionUtil.getStackTrace(e));
893       LOG.error("Error in deleting widget: "+ title);
894     }
895     return result;
896   }
897 
898   public static boolean isDefaultExists() {
899     boolean exists = false;
900     try {
901       getHBaseConnection();
902       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
903       Get dashboardTest = new Get(DASHBOARD_TYPE);
904       dashboardTest.addColumn(COMMON_FAMILY, "default".getBytes(UTF8));
905       exists = table.exists(dashboardTest);
906       table.close();
907     } catch (Exception e) {
908       closeHBase();
909       LOG.error(ExceptionUtil.getStackTrace(e));
910     }
911     return exists;
912   }
913 
914   public static void populateDefaults() {
915     boolean defaultExists = isDefaultExists();
916     try {
917       if(defaultExists) {
918         return;
919       }
920       String hostname = InetAddress.getLocalHost().getHostName().toLowerCase();
921       // Populate example chart widgets
922       String[] metrics = { "SystemMetrics.LoadAverage.1" };
923       createChart("1", "System Load Average", metrics, hostname, "");
924       String[] cpuMetrics = { "SystemMetrics.cpu.combined", "SystemMetrics.cpu.sys", "SystemMetrics.cpu.user" };
925       createChart("2", "CPU Utilization", cpuMetrics, hostname, "percent");
926       String[] memMetrics = { "SystemMetrics.memory.FreePercent", "SystemMetrics.memory.UsedPercent"};
927       createChart("3", "Memory Utilization", memMetrics, hostname, "percent");
928       String[] diskMetrics = { "SystemMetrics.disk.ReadBytes", "SystemMetrics.disk.WriteBytes" };
929       createChart("4", "Disk Utilization", diskMetrics, hostname, "bytes-decimal");
930       String[] netMetrics = { "SystemMetrics.network.TxBytes", "SystemMetrics.network.RxBytes" };
931       createChart("5", "Network Utilization", netMetrics, hostname, "bytes");
932       String[] swapMetrics = { "SystemMetrics.swap.Total", "SystemMetrics.swap.Used", "SystemMetrics.swap.Free" };
933       createChart("6", "Swap Utilization", swapMetrics, hostname, "bytes-decimal");
934       
935       // Namenode heap usage
936       StringBuilder namenode = new StringBuilder();
937       namenode.append(hostname);
938       namenode.append(":NameNode");
939       String[] namenodeHeap = { "HadoopMetrics.jvm.JvmMetrics.MemHeapUsedM", "HadoopMetrics.jvm.JvmMetrics.MemHeapMaxM" };
940       createCircle("7", "Namenode Memory", namenodeHeap, namenode.toString(), "%", "up");
941       
942       // HDFS Usage
943       String[] hdfsUsage = { "HadoopMetrics.dfs.FSNamesystem.CapacityRemainingGB", "HadoopMetrics.dfs.FSNamesystem.CapacityTotalGB" };
944       createCircle("8", "HDFS Remaining", hdfsUsage, hostname, "%", "down");
945 
946       // Resource Manager Memory
947       StringBuilder rmnode = new StringBuilder();
948       rmnode.append(hostname);
949       rmnode.append(":ResourceManager");
950       String[] rmHeap = { "HadoopMetrics.jvm.JvmMetrics.MemHeapUsedM", "HadoopMetrics.jvm.JvmMetrics.MemHeapMaxM" };
951       createCircle("9", "Resource Manager Memory", rmHeap, rmnode.toString(), "%", "up");
952 
953       // Node Managers Health
954       String[] nmh = { "HadoopMetrics.yarn.ClusterMetrics.NumActiveNMs", "HadoopMetrics.yarn.ClusterMetrics.NumLostNMs" };
955       createTile("10", "Node Managers Health", "Node Managers", "Active/Lost", nmh, hostname, "glyphicon-th");
956 
957       // High Availability State
958       String[] ha = { "HadoopMetrics.dfs.FSNamesystem.HAState" };
959       createTile("11", "HDFS High Availability State", "HDFS High Availability", "", ha, hostname, "glyphicon-random");
960 
961       // HDFS Load
962       String[] hdfsLoad = { "HadoopMetrics.dfs.FSNamesystem.TotalLoad" };
963       createTile("12", "HDFS Load Average", "HDFS Load", "", hdfsLoad, hostname, "glyphicon-signal");
964 
965       // Namenode RPC Latency
966       String[] nnLatency = { "HadoopMetrics.rpc.rpc.RpcProcessingTimeAvgTime" };
967       createTile("13", "NameNode Latency", "NameNode RPC Latency", "Milliseconds", nnLatency, hostname, "glyphicon-tasks");
968 
969       // Datanode Health
970       String[] dnHealth = { "HadoopMetrics.dfs.FSNamesystem.StaleDataNodes" };
971       createTile("14", "Datanodes Health", "Datanodes", "Dead", dnHealth, hostname, "glyphicon-hdd");
972 
973       // HBase Master Memory
974       StringBuilder hbaseMaster = new StringBuilder();
975       hbaseMaster.append(hostname);
976       hbaseMaster.append(":Master");
977       String[] hbm = { "HBaseMetrics.jvm.JvmMetrics.MemHeapUsedM", "HBaseMetrics.jvm.JvmMetrics.MemHeapMaxM" };
978       createCircle("15", "HBase Master Memory", hbm, hbaseMaster.toString(), "%", "up");
979 
980       // Demo metrics
981 //      String[] trialAbandonRate = { "HadoopMetrics.jvm.JvmMetrics.MemHeapUsedM", "HadoopMetrics.jvm.JvmMetrics.MemHeapMaxM" };
982 //      createChart("T1", "Trial Abandon Rate", trialAbandonRate, namenode.toString(), "percent");
983 //      createChart("T2", "Unhealthy Clusters", hdfsUsage, hostname, "percent");
984       
985       // Populate default widgets
986       Widget widget = new Widget();
987       widget.setTitle("System Load Average");
988       widget.setSrc(new URI("/hicc/v1/chart/draw/1"));
989       widget.setCol(1);
990       widget.setRow(1);
991       widget.setSize_x(2);
992       widget.setSize_y(1);
993       createWidget(widget);
994 
995       // Populate default dashboard
996       Dashboard dashboard = new Dashboard();
997 
998       widget = new Widget();
999       widget.setTitle("Welcome Page");
1000       widget.setSrc(new URI("/hicc/welcome.html"));
1001       widget.setCol(1);
1002       widget.setRow(1);
1003       widget.setSize_x(10);
1004       widget.setSize_y(5);
1005       createWidget(widget);
1006       dashboard.add(widget);
1007 
1008       widget = new Widget();
1009       widget.setTitle("Trial Downloading");
1010       widget.setSrc(new URI("/hicc/home/downloads.html"));
1011       widget.setCol(1);
1012       widget.setRow(1);
1013       widget.setSize_x(2);
1014       widget.setSize_y(1);
1015       createWidget(widget);
1016 
1017       widget = new Widget();
1018       widget.setTitle("Cluster Running");
1019       widget.setSrc(new URI("/hicc/home/clusters.html"));
1020       widget.setCol(3);
1021       widget.setRow(1);
1022       widget.setSize_x(2);
1023       widget.setSize_y(1);
1024       createWidget(widget);
1025 
1026       widget = new Widget();
1027       widget.setTitle("Users Working");
1028       widget.setSrc(new URI("/hicc/home/users.html"));
1029       widget.setCol(5);
1030       widget.setRow(1);
1031       widget.setSize_x(2);
1032       widget.setSize_y(1);
1033       createWidget(widget);
1034 
1035       widget = new Widget();
1036       widget.setTitle("Applications Running");
1037       widget.setSrc(new URI("/hicc/home/apps.html"));
1038       widget.setCol(7);
1039       widget.setRow(1);
1040       widget.setSize_x(2);
1041       widget.setSize_y(1);
1042       createWidget(widget);
1043 
1044       widget = new Widget();
1045       widget.setTitle("Trial Abandon Rate");
1046       widget.setSrc(new URI("/hicc/v1/circles/draw/11"));
1047       widget.setCol(1);
1048       widget.setRow(2);
1049       widget.setSize_x(2);
1050       widget.setSize_y(2);
1051       createWidget(widget);
1052 
1053       widget = new Widget();
1054       widget.setTitle("Clusters Health");
1055       widget.setSrc(new URI("/hicc/v1/circles/draw/12"));
1056       widget.setCol(3);
1057       widget.setRow(2);
1058       widget.setSize_x(2);
1059       widget.setSize_y(2);
1060       createWidget(widget);
1061 
1062       widget = new Widget();
1063       widget.setTitle("Top Active Clusters");
1064       widget.setSrc(new URI("/hicc/clusters/"));
1065       widget.setCol(5);
1066       widget.setRow(2);
1067       widget.setSize_x(2);
1068       widget.setSize_y(2);
1069       createWidget(widget);
1070 
1071       widget = new Widget();
1072       widget.setTitle("Top Applications");
1073       widget.setSrc(new URI("/hicc/apps/"));
1074       widget.setCol(7);
1075       widget.setRow(2);
1076       widget.setSize_x(2);
1077       widget.setSize_y(2);
1078       createWidget(widget);
1079 
1080       widget = new Widget();
1081       widget.setTitle("Applications Usage");
1082       widget.setSrc(new URI("/hicc/apps/apps-usage.html"));
1083       widget.setCol(7);
1084       widget.setRow(2);
1085       widget.setSize_x(2);
1086       widget.setSize_y(2);
1087       createWidget(widget);
1088 
1089       updateDashboard("default", "", dashboard);
1090 
1091       // Populate user dashboards
1092       dashboard = new Dashboard();
1093 
1094       widget = new Widget();
1095       widget.setTitle("Quick Links");
1096       widget.setSrc(new URI("/hicc/v1/dashboard/quicklinks"));
1097       widget.setCol(1);
1098       widget.setRow(1);
1099       widget.setSize_x(10);
1100       widget.setSize_y(5);
1101       createWidget(widget);
1102       dashboard.add(widget);
1103 
1104       // Log Search widget
1105       widget = new Widget();
1106       widget.setTitle("Log Search");
1107       widget.setSrc(new URI("/hicc/ajax-solr/chukwa"));
1108       widget.setCol(1);
1109       widget.setRow(1);
1110       widget.setSize_x(6);
1111       widget.setSize_y(6);
1112       createWidget(widget);
1113 
1114       // Applications
1115       widget = new Widget();
1116       widget.setTitle("YARN Applications");
1117       widget.setSrc(new URI("http://localhost:8088/"));
1118       widget.setCol(1);
1119       widget.setRow(7);
1120       widget.setSize_x(6);
1121       widget.setSize_y(6);
1122       createWidget(widget);
1123 
1124       // Hadoop Distributed File System
1125       widget = new Widget();
1126       widget.setTitle("HDFS");
1127       widget.setSrc(new URI("http://localhost:50070/explorer.html#/"));
1128       widget.setCol(1);
1129       widget.setRow(7);
1130       widget.setSize_x(6);
1131       widget.setSize_y(6);
1132       createWidget(widget);
1133 
1134       // HBase Tables
1135       widget = new Widget();
1136       widget.setTitle("HBase Tables");
1137       widget.setSrc(new URI("http://localhost:50654/tablesDetailed.jsp"));
1138       widget.setCol(1);
1139       widget.setRow(14);
1140       widget.setSize_x(6);
1141       widget.setSize_y(6);
1142       createWidget(widget);
1143 
1144       widget = new Widget();
1145       widget.setTitle("Top Applications");
1146       widget.setSrc(new URI("/hicc/apps/"));
1147       widget.setCol(1);
1148       widget.setRow(1);
1149       widget.setSize_x(2);
1150       widget.setSize_y(2);
1151       createWidget(widget);
1152 
1153       widget = new Widget();
1154       widget.setTitle("Top Users");
1155       widget.setSrc(new URI("/hicc/users/"));
1156       widget.setCol(1);
1157       widget.setRow(3);
1158       widget.setSize_x(2);
1159       widget.setSize_y(2);
1160       createWidget(widget);
1161       updateDashboard("user", "", dashboard);
1162 
1163       // Populate system dashboards
1164       dashboard = new Dashboard();
1165       widget = new Widget();
1166       widget.setTitle("HDFS High Availability State");
1167       widget.setSrc(new URI("/hicc/v1/tile/draw/11"));
1168       widget.setCol(1);
1169       widget.setRow(1);
1170       widget.setSize_x(2);
1171       widget.setSize_y(1);
1172       createWidget(widget);
1173       dashboard.add(widget);
1174 
1175       widget = new Widget();
1176       widget.setTitle("HDFS Load");
1177       widget.setSrc(new URI("/hicc/v1/tile/draw/12"));
1178       widget.setCol(3);
1179       widget.setRow(1);
1180       widget.setSize_x(2);
1181       widget.setSize_y(1);
1182       createWidget(widget);
1183       dashboard.add(widget);
1184 
1185       widget = new Widget();
1186       widget.setTitle("HDFS Namenode Latency");
1187       widget.setSrc(new URI("/hicc/v1/tile/draw/13"));
1188       widget.setCol(5);
1189       widget.setRow(1);
1190       widget.setSize_x(2);
1191       widget.setSize_y(1);
1192       createWidget(widget);
1193       dashboard.add(widget);
1194 
1195       widget = new Widget();
1196       widget.setTitle("Datanodes Health");
1197       widget.setSrc(new URI("/hicc/v1/tile/draw/14"));
1198       widget.setCol(7);
1199       widget.setRow(1);
1200       widget.setSize_x(2);
1201       widget.setSize_y(1);
1202       createWidget(widget);
1203       dashboard.add(widget);
1204       
1205       widget = new Widget();
1206       widget.setTitle("Node Managers Health");
1207       widget.setSrc(new URI("/hicc/v1/tile/draw/10"));
1208       widget.setCol(9);
1209       widget.setRow(1);
1210       widget.setSize_x(2);
1211       widget.setSize_y(1);
1212       createWidget(widget);
1213       dashboard.add(widget);
1214 
1215       widget = new Widget();
1216       widget.setTitle("HDFS Remaining");
1217       widget.setSrc(new URI("/hicc/v1/circles/draw/8"));
1218       widget.setCol(1);
1219       widget.setRow(2);
1220       widget.setSize_x(2);
1221       widget.setSize_y(2);
1222       createWidget(widget);
1223       dashboard.add(widget);
1224 
1225       widget = new Widget();
1226       widget.setTitle("Namenode Memory");
1227       widget.setSrc(new URI("/hicc/v1/circles/draw/7"));
1228       widget.setCol(3);
1229       widget.setRow(2);
1230       widget.setSize_x(2);
1231       widget.setSize_y(2);
1232       createWidget(widget);
1233       dashboard.add(widget);
1234 
1235       widget = new Widget();
1236       widget.setTitle("Resource Manager Memory");
1237       widget.setSrc(new URI("/hicc/v1/circles/draw/9"));
1238       widget.setCol(5);
1239       widget.setRow(2);
1240       widget.setSize_x(2);
1241       widget.setSize_y(2);
1242       createWidget(widget);
1243       dashboard.add(widget);
1244 
1245       widget = new Widget();
1246       widget.setTitle("HBase Master Memory");
1247       widget.setSrc(new URI("/hicc/v1/circles/draw/15"));
1248       widget.setCol(7);
1249       widget.setRow(2);
1250       widget.setSize_x(2);
1251       widget.setSize_y(2);
1252       createWidget(widget);
1253       dashboard.add(widget);
1254 
1255       widget = new Widget();
1256       widget.setTitle("System Load Average");
1257       widget.setSrc(new URI("/hicc/v1/chart/draw/1"));
1258       widget.setCol(9);
1259       widget.setRow(2);
1260       widget.setSize_x(2);
1261       widget.setSize_y(1);
1262       createWidget(widget);
1263       dashboard.add(widget);
1264 
1265       widget = new Widget();
1266       widget.setTitle("CPU Utilization");
1267       widget.setSrc(new URI("/hicc/v1/chart/draw/2"));
1268       widget.setCol(9);
1269       widget.setRow(3);
1270       widget.setSize_x(2);
1271       widget.setSize_y(1);
1272       createWidget(widget);
1273       dashboard.add(widget);
1274 
1275       widget = new Widget();
1276       widget.setTitle("Memory Utilization");
1277       widget.setSrc(new URI("/hicc/v1/chart/draw/3"));
1278       widget.setCol(9);
1279       widget.setRow(4);
1280       widget.setSize_x(2);
1281       widget.setSize_y(1);
1282       createWidget(widget);
1283       dashboard.add(widget);
1284 
1285       widget = new Widget();
1286       widget.setTitle("Swap Utilization");
1287       widget.setSrc(new URI("/hicc/v1/chart/draw/6"));
1288       widget.setCol(9);
1289       widget.setRow(5);
1290       widget.setSize_x(2);
1291       widget.setSize_y(1);
1292       createWidget(widget);
1293       dashboard.add(widget);
1294 
1295       widget = new Widget();
1296       widget.setTitle("Disk Utilization");
1297       widget.setSrc(new URI("/hicc/v1/chart/draw/4"));
1298       widget.setCol(1);
1299       widget.setRow(4);
1300       widget.setSize_x(4);
1301       widget.setSize_y(2);
1302       createWidget(widget);
1303       dashboard.add(widget);
1304 
1305       widget = new Widget();
1306       widget.setTitle("Network Utilization");
1307       widget.setSrc(new URI("/hicc/v1/chart/draw/5"));
1308       widget.setCol(5);
1309       widget.setRow(4);
1310       widget.setSize_x(4);
1311       widget.setSize_y(2);
1312       createWidget(widget);
1313       dashboard.add(widget);
1314 
1315       // CPU heatmap
1316       widget = new Widget();
1317       widget.setTitle("CPU Heatmap");
1318       widget.setSrc(new URI("/hicc/v1/heatmap/render/SystemMetrics/cpu.combined."));
1319       widget.setCol(1);
1320       widget.setRow(5);
1321       widget.setSize_x(6);
1322       widget.setSize_y(5);
1323       createWidget(widget);
1324 
1325       // HDFS Namenode
1326       widget = new Widget();
1327       widget.setTitle("HDFS UI");
1328       widget.setSrc(new URI("http://localhost:50070/"));
1329       widget.setCol(1);
1330       widget.setRow(11);
1331       widget.setSize_x(6);
1332       widget.setSize_y(6);
1333       createWidget(widget);
1334 
1335       // HBase Master
1336       widget = new Widget();
1337       widget.setTitle("HBase Master UI");
1338       widget.setSrc(new URI("http://localhost:16010/"));
1339       widget.setCol(1);
1340       widget.setRow(18);
1341       widget.setSize_x(6);
1342       widget.setSize_y(6);
1343       createWidget(widget);
1344 
1345 //    widget = new Widget();
1346 //    widget.setTitle("Services Running");
1347 //    widget.setSrc(new URI("/hicc/services/services.html"));
1348 //    widget.setCol(1);
1349 //    widget.setRow(1);
1350 //    widget.setSize_x(2);
1351 //    widget.setSize_y(1);
1352 //    createWidget(widget);
1353 //    dashboard.add(widget);
1354 //    
1355 //    widget = new Widget();
1356 //    widget.setTitle("Applications Running");
1357 //    widget.setSrc(new URI("/hicc/home/apps.html"));
1358 //    widget.setCol(3);
1359 //    widget.setRow(1);
1360 //    widget.setSize_x(2);
1361 //    widget.setSize_y(1);
1362 //    dashboard.add(widget);
1363 
1364 //    widget = new Widget();
1365 //    widget.setTitle("Timeline");
1366 //    widget.setSrc(new URI("/hicc/timeline/"));
1367 //    widget.setCol(7);
1368 //    widget.setRow(2);
1369 //    widget.setSize_x(4);
1370 //    widget.setSize_y(6);
1371 //    createWidget(widget);
1372 //    dashboard.add(widget);
1373 
1374 //      widget = new Widget();
1375 //      widget.setTitle("Alerts");
1376 //      widget.setSrc(new URI("/hicc/alerts/"));
1377 //      widget.setCol(1);
1378 //      widget.setRow(5);
1379 //      widget.setSize_x(6);
1380 //      widget.setSize_y(5);
1381 //      createWidget(widget);
1382 //
1383 //      widget = new Widget();
1384 //      widget.setTitle("Log Errors");
1385 //      widget.setSrc(new URI("/hicc/logs/"));
1386 //      widget.setCol(1);
1387 //      widget.setRow(5);
1388 //      widget.setSize_x(6);
1389 //      widget.setSize_y(5);
1390 //      createWidget(widget);
1391 
1392       updateDashboard("system", "", dashboard);
1393       
1394     } catch (Throwable ex) {
1395       LOG.error(ExceptionUtil.getStackTrace(ex));
1396     }
1397   }
1398 
1399   public static synchronized Dashboard getDashboard(String id, String user) {
1400     Dashboard dash = null;
1401     String key = new StringBuilder().append(id).
1402         append("|").append(user).toString();
1403     try {
1404       getHBaseConnection();
1405       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
1406       Get dashboard = new Get(DASHBOARD_TYPE);
1407       dashboard.addColumn(COMMON_FAMILY, key.getBytes(UTF8));
1408       Result rs = table.get(dashboard);
1409       byte[] buffer = rs.getValue(COMMON_FAMILY, key.getBytes(UTF8));
1410       if(buffer == null) {
1411         // If user dashboard is not found, use default dashboard.
1412         key = new StringBuilder().append(id).append("|").toString();
1413         dashboard = new Get(DASHBOARD_TYPE);
1414         dashboard.addColumn(COMMON_FAMILY, key.getBytes(UTF8));
1415         rs = table.get(dashboard);
1416         buffer = rs.getValue(COMMON_FAMILY, key.getBytes(UTF8));        
1417       }
1418       Gson gson = new Gson();
1419       dash = gson.fromJson(new String(buffer, UTF8), Dashboard.class);
1420       table.close();
1421     } catch (Exception e) {
1422       closeHBase();
1423       LOG.error(ExceptionUtil.getStackTrace(e));
1424       LOG.error("Error retrieving dashboard, id: " + 
1425         id + " user:" + user);
1426     }
1427     return dash;
1428   }
1429 
1430   public static boolean updateDashboard(String id, String user, Dashboard dash) {
1431     boolean result = false;
1432     String key = new StringBuilder().append(id).
1433         append("|").append(user).toString();
1434     try {
1435       getHBaseConnection();
1436       Table table = connection.getTable(TableName.valueOf(CHUKWA_META));
1437       Put put = new Put(DASHBOARD_TYPE);
1438       Gson gson = new Gson();
1439       String buffer = gson.toJson(dash);
1440       put.addColumn(COMMON_FAMILY, key.getBytes(UTF8), buffer.getBytes(UTF8));
1441       table.put(put);
1442       table.close();
1443       result = true;
1444     } catch (Exception e) {
1445       closeHBase();
1446       LOG.error(ExceptionUtil.getStackTrace(e));
1447       LOG.error("Error in updating dashboard, id: " + 
1448         id + " user:" + user);
1449     }
1450     return result;
1451   }
1452 
1453 }