/*
    * Licensed to the Apache Software Foundation (ASF) under one
    * or more contributor license agreements.  See the NOTICE file
    * distributed with this work for additional information
    * regarding copyright ownership.  The ASF licenses this file
    * to you under the Apache License, Version 2.0 (the
    * "License"); you may not use this file except in compliance
    * with the License.  You may obtain a copy of the License at
    *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  package org.apache.hadoop.chukwa.datacollection.writer.parquet;
  
  import java.io.IOException;
  import java.net.InetAddress;
  import java.net.UnknownHostException;
  import java.nio.ByteBuffer;
  import java.util.Calendar;
  import java.util.List;
  
  import org.apache.avro.Schema;
  import org.apache.avro.generic.GenericData;
  import org.apache.avro.generic.GenericRecord;
  import org.apache.hadoop.chukwa.Chunk;
  import org.apache.hadoop.chukwa.datacollection.agent.ChukwaAgent;
  import org.apache.hadoop.chukwa.datacollection.writer.ChukwaWriter;
  import org.apache.hadoop.chukwa.datacollection.writer.PipelineableWriter;
  import org.apache.hadoop.chukwa.datacollection.writer.WriterException;
  import org.apache.hadoop.chukwa.util.ExceptionUtil;
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.log4j.Logger;
  import org.apache.parquet.avro.AvroParquetWriter;
  import org.apache.parquet.hadoop.metadata.CompressionCodecName;
  
  public class ChukwaParquetWriter extends PipelineableWriter {
    private static Logger LOG = Logger.getLogger(ChukwaParquetWriter.class);
    public static final String OUTPUT_DIR_OPT= "chukwaCollector.outputDir";
    private int blockSize = 128 * 1024 * 1024;
    private int pageSize = 1 * 1024 * 1024;
    private Schema avroSchema = null;
    private AvroParquetWriter<GenericRecord> parquetWriter = null;
    protected String outputDir = null;
    private Calendar calendar = Calendar.getInstance();
    private String localHostAddr = null;
    private long rotateInterval = 300000L;
    private long startTime = 0;
    private Path previousPath = null;
    private String previousFileName = null;
    private FileSystem fs = null;
    
    public ChukwaParquetWriter() throws WriterException {
      this(ChukwaAgent.getStaticConfiguration());
    }
  
    public ChukwaParquetWriter(Configuration c) throws WriterException {
      setup(c);
    }
  
    @Override
    public void init(Configuration c) throws WriterException {
    }
  
    private void setup(Configuration c) throws WriterException {
      try {
        localHostAddr = "_" + InetAddress.getLocalHost().getHostName() + "_";
      } catch (UnknownHostException e) {
        localHostAddr = "-NA-";
      }
      outputDir = c.get(OUTPUT_DIR_OPT, "/chukwa/logs");
      blockSize = c.getInt("dfs.blocksize", 128 * 1024 * 1024);
      rotateInterval = c.getLong("chukwaCollector.rotateInterval", 300000L);
      if(fs == null) {
        try {
          fs = FileSystem.get(c);
        } catch (IOException e) {
          throw new WriterException(e);
        }
      }
  
      // load Chukwa Avro schema
      avroSchema = ChukwaAvroSchema.getSchema();
      // generate the corresponding Parquet schema
      rotate();
    }
  
    @Override
    public void close() throws WriterException {
      try {
        parquetWriter.close();
        fs.rename(previousPath, new Path(previousFileName + ".done"));
      } catch (IOException e) {
       throw new WriterException(e);
     }
   }
 
   @Override
   public CommitStatus add(List<Chunk> chunks) throws WriterException {
     long elapsedTime = 0;
     CommitStatus rv = ChukwaWriter.COMMIT_OK;
     for(Chunk chunk : chunks) {
       try {
         GenericRecord record = new GenericData.Record(avroSchema);
         record.put("dataType", chunk.getDataType());
         record.put("data", ByteBuffer.wrap(chunk.getData()));
         record.put("tags", chunk.getTags());
         record.put("seqId", chunk.getSeqID());
         record.put("source", chunk.getSource());
         record.put("stream", chunk.getStreamName());
         parquetWriter.write(record);
         elapsedTime = System.currentTimeMillis() - startTime;
         if(elapsedTime > rotateInterval) {
           rotate();
         }
       } catch (IOException e) {
         LOG.warn("Failed to store data to HDFS.");
         LOG.warn(ExceptionUtil.getStackTrace(e));
       }
     }
     if (next != null) {
       rv = next.add(chunks); //pass data through
     }
     return rv;
   }
   
   private void rotate() throws WriterException {
     if(parquetWriter!=null) {
       try {
         parquetWriter.close();
         String newFileName = previousFileName.substring(0, previousFileName.length() - 7);
         fs.rename(previousPath, new Path(newFileName + ".done"));
       } catch (IOException e) {
         LOG.warn("Fail to close Chukwa write ahead log.");
       }
     }
     startTime = System.currentTimeMillis();
     calendar.setTimeInMillis(startTime);
 
     String newName = new java.text.SimpleDateFormat("yyyyMMddHHmmssSSS")
         .format(calendar.getTime());
     newName += localHostAddr + new java.rmi.server.UID().toString();
     newName = newName.replace("-", "");
     newName = newName.replace(":", "");
     newName = newName.replace(".", "");
     newName = outputDir + "/" + newName.trim() + ".chukwa";
     LOG.info("writing: "+newName);
     Path path = new Path(newName);
     try {
       parquetWriter = new AvroParquetWriter<GenericRecord>(path, avroSchema, CompressionCodecName.SNAPPY, blockSize, pageSize);
       previousPath = path;
       previousFileName = newName;
     } catch (IOException e) {
       throw new WriterException(e);
     }
   }
 
   /**
    * Calculates delay for scheduling the next rotation in case of
    * FixedTimeRotatorScheme. This delay is the time difference between the
    * currentTimestamp (t1) and the next time the collector should rotate the
    * sequence files (t2). t2 is the time when the current rotateInterval ends
    * plus an offset (as set by chukwaCollector.FixedTimeIntervalOffset).
    * So, delay = t2 - t1
    *
    * @param currentTime - the current timestamp
    * @param rotateInterval - chukwaCollector.rotateInterval
    * @param offsetInterval - chukwaCollector.fixedTimeIntervalOffset
    * @return delay for scheduling next rotation
    */
   public long getDelayForFixedInterval(long currentTime, long rotateInterval, long offsetInterval){
     // time since last rounded interval
     long remainder = (currentTime % rotateInterval);
     long prevRoundedInterval = currentTime - remainder;
     long nextRoundedInterval = prevRoundedInterval + rotateInterval;
     long delay = nextRoundedInterval - currentTime + offsetInterval;
 
     if (LOG.isInfoEnabled()) {
       LOG.info("currentTime="+currentTime+" prevRoundedInterval="+
              prevRoundedInterval+" nextRoundedInterval" +
             "="+nextRoundedInterval+" delay="+delay);
     }
 
     return delay;
   }
 }